WLG/datatransfer/data_trans.cpp

#include "data_trans.hpp"
#include "dirent.h"
#include <stdio.h>
#include <zlog.h>

extern zlog_category_t *zct;
extern zlog_category_t *zbt;

DataTrans::DataTrans() : debug_(false) {}

DataTrans::~DataTrans() {}

static int OnDebug(CURL *, curl_infotype itype, char *pData, size_t size, void *) {
    if (itype == CURLINFO_TEXT) {
        zlog_debug(zbt, "[TEXT]%s", pData);
    } else if (itype == CURLINFO_HEADER_IN) {
        zlog_debug(zbt, "[HEADER_IN]%s", pData);
    } else if (itype == CURLINFO_HEADER_OUT) {
        zlog_debug(zbt, "[HEADER_OUT]%s", pData);
    } else if (itype == CURLINFO_DATA_IN) {
        zlog_debug(zbt, "[DATA_IN]%s", pData);
    } else if (itype == CURLINFO_DATA_OUT) {
        zlog_debug(zbt, "[DATA_OUT]%s", pData);
    } else {
        zlog_error(zbt, "invalid itype:%d", itype);        
    }
    return 0;
}

static size_t OnWriteData(void *buffer, size_t size, size_t nmemb, void *lpVoid) {
    std::string *str = dynamic_cast<std::string *>((std::string *)lpVoid);
    if (NULL == str || NULL == buffer) {
		zlog_error(zct, "[OnWriteData] str:%p, buffer:%p", str, buffer);
        return 0;
    }
    char *pData = (char *)buffer;
    str->append(pData, size * nmemb);
    return nmemb;
}

static size_t my_fwrite(void *buffer, size_t size, size_t nmemb, void *stream) {
    zlog_debug(zct, "size = %d", size);
    struct DownloadFile *out = (struct DownloadFile *)stream;
    if (out && !out->stream) {
        out->stream = fopen(out->filename, "wb");
        if (!out->stream) {
            zlog_error(zct, "[my_fwrite] fail to open:%s", out->filename);
            return 0;
        }
    }
    return fwrite(buffer, size, nmemb, out->stream);
}

int DataTrans::download(char *pFilename, std::string &strUrl, std::string &strResponse, bool bDownload) {
    CURL *curl = NULL;
    CURLcode res;
    struct DownloadFile dlfile = {pFilename, NULL}; //定义下载到本地的文件位置和路径
    curl_global_init(CURL_GLOBAL_DEFAULT);
    curl = curl_easy_init();
    if (curl == NULL) {
        zlog_error(zct, "[download] curl_easy_init failed.");
        return 1;
    } 
             
    curl_easy_setopt(curl, CURLOPT_URL, strUrl.c_str());
    if (bDownload) {
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, my_fwrite); //当有数据被写入，回调函数被调用，
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &dlfile);       //设置结构体的指针传递给回调函数
    } else {
        curl_easy_setopt(curl, CURLOPT_READFUNCTION, NULL);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, OnWriteData);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&strResponse);
    }
    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 8); //连接超时，这个数值如果设置太短可能导致数据请求不到就断开了
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10);       //接收数据时超时设置，如果10秒内数据未接收完，直接退出
    curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
    curl_easy_setopt(curl, CURLOPT_USERPWD, "SUREN:SUREN");

    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);
    if (res != CURLE_OK) {
        curl_global_cleanup();
        zlog_error(zct, "[download] curl_easy_perform ret:%s", curl_easy_strerror(res));
        return -1;
    }
    
    zlog_debug(zct, "[download] strResponse = %s", strResponse.c_str());
    if (bDownload) {
        if (dlfile.stream) {
            fclose(dlfile.stream);
        }
    }

    curl_global_cleanup();
    return 0;
}

size_t writedata2file(void *ptr, size_t size, size_t nmemb, FILE *stream) {
    size_t written = fwrite(ptr, size, nmemb, stream);
    return written;
}

int DataTrans::dl_curl_post_req(const std::string &url, const std::string &postParams, std::string &filename) {
    CURL *curl = nullptr;
    FILE *fp = nullptr;
    CURLcode res;

    res = curl_global_init(CURL_GLOBAL_ALL);
    if (CURLE_OK != res) {
        zlog_error(zct, "[dl_curl_post_req] curl_global_init failed.");
        curl_global_cleanup();
        return 1;
    }

    curl = curl_easy_init();
	if (curl == NULL) {
        zlog_error(zct, "[dl_curl_post_req] curl_easy_init failed.");
        curl_global_cleanup();
        return 2;
    }    
	
	res = curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
	if (res != CURLE_OK) {	
        zlog_error(zct, "[dl_curl_post_req] curl_easy_setopt CURLOPT_URL failed.");		
		curl_easy_cleanup(curl);
		curl_global_cleanup();
		return 3;
	}

	res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writedata2file);
	if (res != CURLE_OK) {		
        zlog_error(zct, "[dl_curl_post_req] curl_easy_setopt CURLOPT_WRITEFUNCTION failed.");
		curl_easy_cleanup(curl);
		curl_global_cleanup();
		return 4;
	}

    fp = fopen(filename.c_str(), "wb");
    if (fp == nullptr) {
        zlog_error(zct, "[dl_curl_post_req] fopen:%s failed", filename.c_str());
        curl_easy_cleanup(curl);
		curl_global_cleanup();
        return 5;
    }

	res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
	if (res != CURLE_OK) {
        zlog_error(zct, "[dl_curl_post_req] curl_easy_setopt CURLOPT_WRITEDATA failed.");
		fclose(fp);
		curl_easy_cleanup(curl);
		curl_global_cleanup();
		return 6;
	}

	res = curl_easy_perform(curl);
	fclose(fp);
	if (res != CURLE_OK) {
		zlog_error(zct, "[dl_curl_post_req] curl_easy_perform() failed: %s", curl_easy_strerror(res));
		curl_easy_cleanup(curl);
		curl_global_cleanup();
		return 7;
	}

	curl_easy_cleanup(curl);
    curl_global_cleanup();
    return 0;
}

int DataTrans::Post(const std::string &strUrl, const std::string &strPost, std::string &strResponse) {
    CURLcode res;
    CURL *curl = curl_easy_init();
    if (NULL == curl) {
		zlog_error(zct, "[Post] curl_easy_init failed.");
        return CURLE_FAILED_INIT;
    }
    if (debug_) {
        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
        curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, OnDebug);
    }
    curl_easy_setopt(curl, CURLOPT_URL, strUrl.c_str());
    curl_easy_setopt(curl, CURLOPT_POST, 1);
    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, strPost.c_str());
    curl_easy_setopt(curl, CURLOPT_READFUNCTION, NULL);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, OnWriteData);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&strResponse);
    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3);
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3);
    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);
    if (res != CURLE_OK) {
		zlog_error(zct, "[Post] curl_easy_perform() failed: %s", curl_easy_strerror(res));		
		return 1;
	}
    return res;
}

int DataTrans::Get(const std::string &strUrl, std::string &strResponse) {
    CURLcode res;
    CURL *curl = curl_easy_init();
    if (NULL == curl) {
		zlog_error(zct, "[Get] curl_easy_init failed.");
        return CURLE_FAILED_INIT;
    }
    if (debug_) {
        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
        curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, OnDebug);
    }
    curl_easy_setopt(curl, CURLOPT_URL, strUrl.c_str());
    curl_easy_setopt(curl, CURLOPT_READFUNCTION, NULL);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, OnWriteData);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&strResponse);
    /**
    * 当多个线程都使用超时处理的时候，同时主线程中有sleep或是wait等操作。
    * 如果不设置这个选项，libcurl将会发信号打断这个wait从而导致程序退出。
    */

    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3);
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3);
    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);
    if (res != CURLE_OK) {
		zlog_error(zct, "[Get] curl_easy_perform() failed: %s", curl_easy_strerror(res));		
		return 1;
	}
    return res;
}

int DataTrans::Posts(const std::string &strUrl, const std::string &strPost, std::string &strResponse, const char *pCaPath) {
    CURLcode res;
    CURL *curl = curl_easy_init();
    if (NULL == curl) {
		zlog_error(zct, "[Posts] curl_easy_init failed.");
        return CURLE_FAILED_INIT;
    }

    if (debug_) {
        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
        curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, OnDebug);
    }
    curl_easy_setopt(curl, CURLOPT_URL, strUrl.c_str());
    curl_easy_setopt(curl, CURLOPT_POST, 1);
    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, strPost.c_str());
    curl_easy_setopt(curl, CURLOPT_READFUNCTION, NULL);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, OnWriteData);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&strResponse);
    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
    if (NULL == pCaPath) {
        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, false);
    } else {
        // 缺省情况就是PEM，所以无需设置，另外支持DER
        // curl_easy_setopt(curl,CURLOPT_SSLCERTTYPE,"PEM");
        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, true);
        curl_easy_setopt(curl, CURLOPT_CAINFO, pCaPath);
    }
    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3);
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3);
    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);
    if (res != CURLE_OK) {
		zlog_error(zct, "[Posts] curl_easy_perform() failed: %s", curl_easy_strerror(res));		
		return 1;
	}
    return res;
}

int DataTrans::Gets(const std::string &strUrl, std::string &strResponse, const char *pCaPath) {
    CURLcode res;
    CURL *curl = curl_easy_init();
    if (NULL == curl) {
		zlog_error(zct, "[Gets] curl_easy_init failed.");
        return CURLE_FAILED_INIT;
    }
    if (debug_) {
        curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
        curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, OnDebug);
    }
    curl_easy_setopt(curl, CURLOPT_URL, strUrl.c_str());
    curl_easy_setopt(curl, CURLOPT_READFUNCTION, NULL);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, OnWriteData);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&strResponse);
    curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
    if (NULL == pCaPath) {
        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, false);
    } else {
        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, true);
        curl_easy_setopt(curl, CURLOPT_CAINFO, pCaPath);
    }
    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 3);
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3);
    res = curl_easy_perform(curl);
    curl_easy_cleanup(curl);
    if (res != CURLE_OK) {
		zlog_error(zct, "[Gets] curl_easy_perform() failed: %s", curl_easy_strerror(res));		
		return 1;
	}
    return res;
}

int DataTrans::upload_file(const std::string &strUrl, const std::string &filename, std::string &response) {
    CURL *curl;
    CURLM *multi_handle;
    int still_running;
    char res[60];
    struct curl_httppost *formpost = NULL;
    struct curl_httppost *lastptr = NULL;
    struct curl_slist *headerlist = NULL;
    static const char buf[] = "Expect:";
    curl_global_init(CURL_GLOBAL_ALL);
    /* Fill in the file upload field. This makes libcurl load data from
    the given file name when curl_easy_perform() is called. */

    /* Fill in the filename field */
    curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "filename", CURLFORM_FILE, filename.c_str(), CURLFORM_END);
    curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "submit", CURLFORM_COPYCONTENTS, "upload", CURLFORM_END);

    curl = curl_easy_init();
	if (NULL == curl) {
		zlog_error(zct, "[upload_file] curl_easy_init failed.");
        return CURLE_FAILED_INIT;
    }

    multi_handle = curl_multi_init();
	if (NULL == multi_handle) {
		zlog_error(zct, "[upload_file] curl_multi_init failed.");
		curl_easy_cleanup(curl);
        return CURLE_FAILED_INIT;
    }
    /* initalize custom header list (stating that Expect: 100-continue is not wanted */
    headerlist = curl_slist_append(headerlist, buf);
    
	/* what URL that receives this POST */
	curl_easy_setopt(curl, CURLOPT_URL, strUrl.c_str());
	curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_DIGEST);
	curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, res);
	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist);
	curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
	curl_multi_add_handle(multi_handle, curl);
	CURLMcode ret1 = curl_multi_perform(multi_handle, &still_running);
    if (ret1 != CURLM_OK) {
		zlog_error(zct, "[upload_file] curl_multi_perform() failed: %d", ret1);
		return ret1;
	}

	do {
		struct timeval timeout;
		int rc;
		fd_set fdread;
		fd_set fdwrite;
		fd_set fdexcep;
		int maxfd = -1;
		long curl_timeo = -1;
		FD_ZERO(&fdread);
		FD_ZERO(&fdwrite);
		FD_ZERO(&fdexcep);
		/* set a suitable timeout to play around with */
		timeout.tv_sec = 1;
		timeout.tv_usec = 0;
		curl_multi_timeout(multi_handle, &curl_timeo);
		if (curl_timeo >= 0) {
			timeout.tv_sec = curl_timeo / 1000;
			if (timeout.tv_sec > 1)
				timeout.tv_sec = 1;
			else
				timeout.tv_usec = (curl_timeo % 1000) * 1000;
		}

		/* get file descriptors from the transfers */
		curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
		/* In a real-world program you OF COURSE check the return code of the
		function calls.  On success, the value of maxfd is guaranteed to be
		greater or equal than -1.  We call select(maxfd + 1, ...), specially in
		case of (maxfd == -1), we call select(0, ...), which is basically equal
		to sleep. */
		rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
		switch (rc) {
			case -1: break;
			case 0:
			default:
				/* timeout or readable/writable sockets */
				zlog_info(zct, "perform!");
				curl_multi_perform(multi_handle, &still_running);
				zlog_info(zct, "running: %d!", still_running);
				break;
		}
	} while (still_running);

	curl_multi_cleanup(multi_handle);
	curl_easy_cleanup(curl);
	curl_formfree(formpost);
	curl_slist_free_all(headerlist);    
    return 0;
}

void DataTrans::SetDebug(bool bDebug) { debug_ = bDebug; }