共计 8399 个字符,预计需要花费 21 分钟才能阅读完成。
最近要写一个代理程序,软件最终要跑在嵌入式设备上,其中一部分是需要做一个简单爬虫程序,用来和嵌入式设备上的Web做信息交互。我不想用第三方的任何库,如是简单看了下http协议,用一天时间实现了http协议的客户端,实现Get、Post、UpFile(文件上传)等常用操作,需要完善的部分是Cookie没有自动提取和传输,需要自己手动处理,朋友们可以完善吧!写个日志,便于日后参考!希望对朋友有参考。
由于最终要在嵌入式设备上运行,所以用 #define Windows 来区分,Socket部分Window和Linux还是有差别的,程序兼容了Windows和Liunx。
通信类头文件
#pragma once #include<string> #ifndef WINDOWS #define WINDOWS #endif #ifdef WINDOWS #include<WINSOCK2.H> #include <WS2tcpip.h> #else #include <iconv.h> #include <netinet/in.h> #include <stdio.h> #include <sys/types.h> #include <sys/socket.h> #include <arpa/inet.h> #include <string.h> #include <stdlib.h> #include <sys/stat.h> #include <unistd.h> #include <fcntl.h> #endif // WINDOWS using namespace std; class TCPClient { public: TCPClient(string host); ~TCPClient(); bool ConnectToServer(); bool SendData(const char * sendBuffer,int dataLen); string RecvData(); private: #ifdef WINDOWS SOCKET clientSocket; #else int clientSocket; #endif string hostAddr; bool isConnected; };
封装基础通信类cpp
#include "TCPClient.h" #include<vector> #include<errno.h> #ifdef WINDOWS #pragma warning(disable:4996) #pragma comment(lib, "WS2_32.lib") #endif // WINDOWS #ifdef WINDOWS TCPClient::TCPClient(string host) { hostAddr = host; isConnected = false; #ifdef WINDOWS WSAData wsdata; if (WSAStartup(MAKEWORD(2, 2), &wsdata) != 0) { WSACleanup(); printf("WSAStartup failured \r\n"); } else { printf("WSAStartup OK\r\n"); } #endif } bool TCPClient::ConnectToServer() { if (isConnected) return true; clientSocket = socket(AF_INET, SOCK_STREAM, 0); if (clientSocket == -1) { printf("sockclient create fail ! \n"); return false; } struct sockaddr_in dest_addr; /* 目的地址*/ dest_addr.sin_family = AF_INET; /* host byte order */ dest_addr.sin_port = htons(80); /* short, network byte order */ #ifdef WINDOWS dest_addr.sin_addr.s_addr = inet_addr(this->hostAddr.c_str()); #else inet_pton(AF_INET, this->hostAddr.c_str(), &dest_addr.sin_addr.s_addr); #endif // WINDOWS if (connect(clientSocket, (struct sockaddr *)&dest_addr, sizeof(struct sockaddr)) < 0) { #ifdef WINDOWS closesocket(clientSocket); printf("connect controlServer failured%d\r\n", GetLastError()); #else close(clientSocket); printf("connect controlServer failured\r\n"); #endif return false; } isConnected = true; return true; } TCPClient::~TCPClient() { #ifdef WINDOWS closesocket(clientSocket); WSACleanup(); #else close(clientSocket); #endif } bool TCPClient::SendData(const char *sendBuffer, int dataLen) { if (!ConnectToServer()) return false; int sendLen = 0; int oneLen = 0; while (sendLen < dataLen) { oneLen = send(this->clientSocket, sendBuffer + sendLen, dataLen - sendLen, 0); if (oneLen <= 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { //这几种错误码,认为连接是正常的,继续接收 } else { //printf("%d", WSAGetLastError()); return 0; } break; } else sendLen += oneLen; } return true; } string TCPClient::RecvData() { char recvData[1000] = { 0 }; string responseData = ""; int recLen = 0; int recLenOfAll = 0; while ((recLen = recv(this->clientSocket, recvData, sizeof(recvData) - 1, 0)) > 0) { responseData.append(recvData); memset(recvData, 0, 1000); //ZeroMemory(recvData, 1000); recLenOfAll += recLen; } string resp; #ifdef WINDOWS resp =responseData.c_str(); closesocket(this->clientSocket); #else resp = (responseData.c_str()); close(this->clientSocket); #endif isConnected = false; return resp; }
WebRequest头文件
#pragma once #include<iostream> #include<thread> #include<mutex> #include<map> #include<string> #include "TCPClient.h" using namespace std; enum RequestType { Post, Get }; class WebRequest { private: string SendData(string method, string path, map<string, string> paramters, string referUrl); string SendData(string method, string path, string bodyInfo, string referUrl); string SendData(string method, string path, string referUrl); public: WebRequest(string host); ~WebRequest(); string GetData(string path, map<string, string> paramters, string referUrl); string GetData(string path, string referUrl); string PostData(string path, map<string, string> paramters, string referUrl); string PostData(string path, string referUrl); string GetData(string path, string bodyInfo, string referUrl); string PostData(string path, string bodyInfo, string referUrl); string UpFile(string path, string FileName); public: map<string, string> Headers; RequestType requestType; private: string GetHeadContent(); TCPClient *tcpClient; string hostAddr; };
webrequest.cpp
#include "WebRequest.h" #include<vector> #include<fstream> WebRequest::WebRequest(string host) { this->hostAddr = host; //Headers.insert(pair<string,string>("A", "B")); map<string, string> *HeadersTest = new map<string, string>(); Headers["Accept"]="*/*"; Headers["Host"] = host; Headers["Accept-Language"] = "zh-CN"; Headers["Accept-Encoding"] = "gzip, deflate"; Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"; Headers["Connection"] = "Keep-Alive"; tcpClient = new TCPClient(host); } WebRequest::~WebRequest() { delete tcpClient; } string WebRequest::GetHeadContent() { string headContent = ""; for (const auto& kv : this->Headers) { if (kv.second.size() == 0) continue; headContent.append(kv.first); headContent.append(": "); headContent.append(kv.second); headContent.append("\r\n"); } return headContent; } string WebRequest::SendData( string method, string path, string bodyInfo, string referUrl) { string headStr = ""; headStr.append(method+" " + path + " HTTP/1.1\r\n"); headStr.append(GetHeadContent()); if (referUrl.length() > 0) headStr.append("Referer: http://"+hostAddr + referUrl +"\r\n"); headStr.append("Content-Length: " + to_string(bodyInfo.size())); headStr.append("\r\n\r\n"); headStr.append(bodyInfo); if (tcpClient->SendData(headStr.c_str(), headStr.length())) return tcpClient->RecvData(); else return "发送请求失败!"; } string WebRequest::GetData(string path, map<string, string> paramters, string referUrl) { string recstr = this->SendData("GET", path, paramters, referUrl); //this->temRecStr = recstr; return recstr; } string WebRequest::PostData(string path, map<string, string> paramters, string referUrl) { return this->SendData("POST", path, paramters, referUrl); } string WebRequest::GetData(string path, string referUrl) { return this->SendData("GET", path, referUrl); } string WebRequest::PostData(string path, string referUrl) { return this->SendData("POST", path, referUrl); } string WebRequest::GetData(string path, string bodyInfo, string referUrl) { return this->SendData("GET", path, bodyInfo, referUrl); } string WebRequest::PostData(string path, string bodyInfo, string referUrl) { return this->SendData("POST", path, bodyInfo, referUrl); } string WebRequest::SendData(string method, string path, map<string,string> paramters,string referUrl) { string headStr = ""; headStr.append(method+" "+ path+" HTTP/1.1\r\n"); headStr.append(GetHeadContent()); if (referUrl.length() > 0) headStr.append("Referer: http://" +hostAddr+ referUrl + "\r\n"); string bodyInfo = ""; for (const auto& kv : paramters) { //if(TString::endsWith(path,"&")) if (bodyInfo.length() > 0) bodyInfo.append("&"); bodyInfo.append(kv.first); bodyInfo.append("="); bodyInfo.append(kv.second); } headStr.append("Content-Length: " + to_string(bodyInfo.size())); headStr.append("\r\n\r\n"); headStr.append(bodyInfo); if (this->tcpClient->SendData(headStr.c_str(), headStr.length())) return tcpClient->RecvData(); else return"请求失败!"; } string WebRequest::SendData(string method, string path, string referUrl) { string headStr = ""; headStr.append(method + " " + path + " HTTP/1.1\r\n"); headStr.append(GetHeadContent()); if (referUrl.length() > 0) headStr.append("Referer: http://" + hostAddr + referUrl + "\r\n"); headStr.append("Content-Length:0" ); headStr.append("\r\n\r\n"); if (this->tcpClient->SendData(headStr.c_str(), headStr.length())) return tcpClient->RecvData(); else return"请求失败!"; } string WebRequest::UpFile(string path, string fileName) { string boundary = "---------------------------7d33a816d302b6\r\n"; this->Headers["Content-Type"] = "multipart/form-data;boundary=---------------------------7d33a816d302b6";// +boundary; string bodyInfo = ""; fstream fs; fs.open(fileName, ios::binary | ios::in); if (!fs) { cout << "文件不存在!" << fileName << endl; return "上传文件不存在!"; } fs.seekg(0, fs.end); int fsLen=fs.tellg(); bodyInfo.append(boundary);//文件内容开始, bodyInfo.append("Content-Disposition: form-data; name=\"fx_19V_11.10.128.14.uot\"; filename=\"" + fileName + "\"\r\n"); bodyInfo.append("Content-Type: application/octet-stream\r\n"); string sendContent = ""; sendContent.append("POST " + path + " HTTP/1.1\r\n"); sendContent.append(GetHeadContent()); sendContent.append("Content-Length: " + to_string(fsLen + bodyInfo.length() + boundary.length())); sendContent.append("\r\n\r\n"); sendContent.append(bodyInfo); const char* sendBuffer = sendContent.c_str(); if (!tcpClient->SendData(sendBuffer, sendContent.length())) return "发送请求失败"; //这里读取文件,发送文件 fs.seekg(0, fs.beg); char fsBuffer[2000]; int readLen = 0; int sendAll = 0; while (!fs.eof()) { fs.read(fsBuffer, 2000); readLen=fs.gcount(); if (!tcpClient->SendData(fsBuffer, readLen)) break; else { sendAll += readLen; cout <<"已发送:"<< sendAll << endl; } } fs.close(); if(!tcpClient->SendData(("\r\n"+boundary).c_str(), boundary.length()))//协议结束 return "发送请求失败"; return tcpClient->RecvData(); }
正文完