libcurl的封装，支持同步异步请求，支持多线程下载，支持https
rbyt
9年前
最近在做一个项目，需要用到http get post等
需求分析需要做到同步和异步，异步请求的返回以可选的回调通知的方式进行。
本人以Linux为例，一步一步的来实现。
配置并且编译libcurl我以在Linux底下的交叉编译举例。
libcurl源码下载: http://curl.haxx.se/download.html
配置libcurl支持https和zlib压缩，必须需要openssl和zlib库
openssl库源码下载: http://www.openssl.org/source/。下载1.02a以上的版本，避开心脏出血漏洞。
zlib源码下载：http://www.zlib.net/。下载最新版本代码。
新建文件夹carbon。源码解压至目录carbon。
1.1 配置openssl并且编译
配置和编译脚本:
#!/bin/bash # Cross-compile environment for Android on ARMv7 and x86 # # Contents licensed under the terms of the OpenSSL license # http://www.openssl.org/source/license.html # # See http://wiki.openssl.org/index.php/FIPS_Library_and_Android # and http://wiki.openssl.org/index.php/Android ##################################################################### # Set ANDROID_NDK_ROOT to you NDK location. For example, # /opt/android-ndk-r8e or /opt/android-ndk-r9. This can be done in a # login script. If ANDROID_NDK_ROOT is not specified, the script will # try to pick it up with the value of _ANDROID_NDK_ROOT below. If # ANDROID_NDK_ROOT is set, then the value is ignored. # _ANDROID_NDK="android-ndk-r8e" #_ANDROID_NDK="android-ndk-r9" _ANDROID_NDK="android-ndk-r10" ANDROID_NDK_ROOT=$HOME/ndk/android-ndk-r10d # Set _ANDROID_EABI to the EABI you want to use. You can find the # list in $ANDROID_NDK_ROOT/toolchains. This value is always used. # _ANDROID_EABI="x86-4.6" # _ANDROID_EABI="arm-linux-androideabi-4.6" _ANDROID_EABI="arm-linux-androideabi-4.8" export ROOTDIR="${PWD}" # Set _ANDROID_ARCH to the architecture you are building for. # This value is always used. # _ANDROID_ARCH=arch-x86 _ANDROID_ARCH=arch-arm # Set _ANDROID_API to the API you want to use. You should set it # to one of: android-14, android-9, android-8, android-14, android-5 # android-4, or android-3. You can't set it to the latest (for # example, API-17) because the NDK does not supply the platform. At # Android 5.0, there will likely be another platform added (android-22?). # This value is always used. # _ANDROID_API="android-14" # _ANDROID_API="android-18" # _ANDROID_API="android-19" _ANDROID_API="android-5" ##################################################################### # If the user did not specify the NDK location, try and pick it up. # We expect something like ANDROID_NDK_ROOT=/opt/android-ndk-r8e # or ANDROID_NDK_ROOT=/usr/local/android-ndk-r8e. if [ -z "$ANDROID_NDK_ROOT" ]; then _ANDROID_NDK_ROOT="" if [ -z "$_ANDROID_NDK_ROOT" ] && [ -d "/usr/local/$_ANDROID_NDK" ]; then _ANDROID_NDK_ROOT="/usr/local/$_ANDROID_NDK" fi if [ -z "$_ANDROID_NDK_ROOT" ] && [ -d "/opt/$_ANDROID_NDK" ]; then _ANDROID_NDK_ROOT="/opt/$_ANDROID_NDK" fi if [ -z "$_ANDROID_NDK_ROOT" ] && [ -d "$HOME/$_ANDROID_NDK" ]; then _ANDROID_NDK_ROOT="$HOME/$_ANDROID_NDK" fi if [ -z "$_ANDROID_NDK_ROOT" ] && [ -d "$PWD/$_ANDROID_NDK" ]; then _ANDROID_NDK_ROOT="$PWD/$_ANDROID_NDK" fi # If a path was set, then export it if [ ! -z "$_ANDROID_NDK_ROOT" ] && [ -d "$_ANDROID_NDK_ROOT" ]; then export ANDROID_NDK_ROOT="$_ANDROID_NDK_ROOT" fi fi # Error checking # ANDROID_NDK_ROOT should always be set by the user (even when not running this script) # http://groups.google.com/group/android-ndk/browse_thread/thread/a998e139aca71d77 if [ -z "$ANDROID_NDK_ROOT" ] || [ ! -d "$ANDROID_NDK_ROOT" ]; then echo "Error: ANDROID_NDK_ROOT is not a valid path. Please edit this script." # echo "$ANDROID_NDK_ROOT" # exit 1 fi # Error checking if [ ! -d "$ANDROID_NDK_ROOT/toolchains" ]; then echo "Error: ANDROID_NDK_ROOT/toolchains is not a valid path. Please edit this script." # echo "$ANDROID_NDK_ROOT/toolchains" # exit 1 fi # Error checking if [ ! -d "$ANDROID_NDK_ROOT/toolchains/$_ANDROID_EABI" ]; then echo "Error: ANDROID_EABI is not a valid path. Please edit this script." # echo "$ANDROID_NDK_ROOT/toolchains/$_ANDROID_EABI" # exit 1 fi ##################################################################### # Based on ANDROID_NDK_ROOT, try and pick up the required toolchain. We expect something like: # /opt/android-ndk-r83/toolchains/arm-linux-androideabi-4.7/prebuilt/linux-x86_64/bin # Once we locate the toolchain, we add it to the PATH. Note: this is the 'hard way' of # doing things according to the NDK documentation for Ice Cream Sandwich. # https://android.googlesource.com/platform/ndk/+/ics-mr0/docs/STANDALONE-TOOLCHAIN.html ANDROID_TOOLCHAIN="" for host in "linux-x86_64" "linux-x86" "darwin-x86_64" "darwin-x86" do if [ -d "$ANDROID_NDK_ROOT/toolchains/$_ANDROID_EABI/prebuilt/$host/bin" ]; then ANDROID_TOOLCHAIN="$ANDROID_NDK_ROOT/toolchains/$_ANDROID_EABI/prebuilt/$host/bin" break fi done # Error checking if [ -z "$ANDROID_TOOLCHAIN" ] || [ ! -d "$ANDROID_TOOLCHAIN" ]; then echo "Error: ANDROID_TOOLCHAIN is not valid. Please edit this script." # echo "$ANDROID_TOOLCHAIN" # exit 1 fi case $_ANDROID_ARCH in arch-arm) ANDROID_TOOLS="arm-linux-androideabi-gcc arm-linux-androideabi-ranlib arm-linux-androideabi-ld" ;; arch-x86) ANDROID_TOOLS="i686-linux-android-gcc i686-linux-android-ranlib i686-linux-android-ld" ;; *) echo "ERROR ERROR ERROR" ;; esac for tool in $ANDROID_TOOLS do # Error checking if [ ! -e "$ANDROID_TOOLCHAIN/$tool" ]; then echo "Error: Failed to find $tool. Please edit this script." # echo "$ANDROID_TOOLCHAIN/$tool" # exit 1 fi done # Only modify/export PATH if ANDROID_TOOLCHAIN good if [ ! -z "$ANDROID_TOOLCHAIN" ]; then export ANDROID_TOOLCHAIN="$ANDROID_TOOLCHAIN" export PATH="$ANDROID_TOOLCHAIN":"$PATH" fi ##################################################################### # For the Android SYSROOT. Can be used on the command line with --sysroot # https://android.googlesource.com/platform/ndk/+/ics-mr0/docs/STANDALONE-TOOLCHAIN.html export ANDROID_SYSROOT="$ANDROID_NDK_ROOT/platforms/$_ANDROID_API/$_ANDROID_ARCH" export SYSROOT="$ANDROID_SYSROOT" export NDK_SYSROOT="$ANDROID_SYSROOT" # Error checking if [ -z "$ANDROID_SYSROOT" ] || [ ! -d "$ANDROID_SYSROOT" ]; then echo "Error: ANDROID_SYSROOT is not valid. Please edit this script." # echo "$ANDROID_SYSROOT" # exit 1 fi ##################################################################### # If the user did not specify the FIPS_SIG location, try and pick it up # If the user specified a bad location, then try and pick it up too. if [ -z "$FIPS_SIG" ] || [ ! -e "$FIPS_SIG" ]; then # Try and locate it _FIPS_SIG="" if [ -d "/usr/local/ssl/$_ANDROID_API" ]; then _FIPS_SIG=`find "/usr/local/ssl/$_ANDROID_API" -name incore` fi if [ ! -e "$_FIPS_SIG" ]; then _FIPS_SIG=`find $PWD -name incore` fi # If a path was set, then export it if [ ! -z "$_FIPS_SIG" ] && [ -e "$_FIPS_SIG" ]; then export FIPS_SIG="$_FIPS_SIG" fi fi # Error checking. Its OK to ignore this if you are *not* building for FIPS if [ -z "$FIPS_SIG" ] || [ ! -e "$FIPS_SIG" ]; then echo "Error: FIPS_SIG does not specify incore module. Please edit this script." # echo "$FIPS_SIG" # exit 1 fi ##################################################################### # Most of these should be OK (MACHINE, SYSTEM, ARCH). RELEASE is ignored. export MACHINE=armv7 export RELEASE=2.6.37 export SYSTEM=android export ARCH=arm export CROSS_COMPILE="arm-linux-androideabi-" if [ "$_ANDROID_ARCH" == "arch-x86" ]; then export MACHINE=i686 export RELEASE=2.6.37 export SYSTEM=android export ARCH=x86 export CROSS_COMPILE="i686-linux-android-" fi # For the Android toolchain # https://android.googlesource.com/platform/ndk/+/ics-mr0/docs/STANDALONE-TOOLCHAIN.html export ANDROID_SYSROOT="$ANDROID_NDK_ROOT/platforms/$_ANDROID_API/$_ANDROID_ARCH" export SYSROOT="$ANDROID_SYSROOT" export NDK_SYSROOT="$ANDROID_SYSROOT" export ANDROID_NDK_SYSROOT="$ANDROID_SYSROOT" export ANDROID_API="$_ANDROID_API" # CROSS_COMPILE and ANDROID_DEV are DFW (Don't Fiddle With). Its used by OpenSSL build system. # export CROSS_COMPILE="arm-linux-androideabi-" export ANDROID_DEV="$ANDROID_NDK_ROOT/platforms/$_ANDROID_API/$_ANDROID_ARCH/usr" export HOSTCC=gcc VERBOSE=1 if [ ! -z "$VERBOSE" ] && [ "$VERBOSE" != "0" ]; then echo "ANDROID_NDK_ROOT: $ANDROID_NDK_ROOT" echo "ANDROID_ARCH: $_ANDROID_ARCH" echo "ANDROID_EABI: $_ANDROID_EABI" echo "ANDROID_API: $ANDROID_API" echo "ANDROID_SYSROOT: $ANDROID_SYSROOT" echo "ANDROID_TOOLCHAIN: $ANDROID_TOOLCHAIN" echo "FIPS_SIG: $FIPS_SIG" echo "CROSS_COMPILE: $CROSS_COMPILE" echo "ANDROID_DEV: $ANDROID_DEV" fi cd openssl if [ $# -gt 0 ]; then perl -pi -e 's/install: all install_docs install_sw/install: install_docs install_sw/g' Makefile.org ./config -DOPENSSL_NO_HEARTBEATS no-shared no-ssl2 no-ssl3 no-comp no-hw no-engine --openssldir=${ROOTDIR}/build/openssl fi make depend make && make install openssl configure1.2 配置zlib并且编译
配置脚本:
#!/bin/sh export ROOTDIR="${PWD}" cd zlib/ export CROSS_COMPILE="arm-linux-androideabi" export CPPFLAGS="-fPIC" export CFLAGS="-fPIC" export AR=${CROSS_COMPILE}-ar export AS=${CROSS_COMPILE}-as export LD=${CROSS_COMPILE}-ld export RANLIB=${CROSS_COMPILE}-ranlib export CC=${CROSS_COMPILE}-gcc export CXX=${CROSS_COMPILE}-g++ export NM=${CROSS_COMPILE}-nm ./configure --prefix=${ROOTDIR}/build/zlib --static zlib configure配置成功之后，cd进代码目录执行make && make install命令即可
1.3 配置libcurl并且编译
#!/bin/sh export ROOTDIR="${PWD}" cd curl-7.42.1/ export CROSS_COMPILE="arm-linux-androideabi" export CPPFLAGS="-fPIC -I${ROOTDIR}/build/openssl/include -I${ROOTDIR}/build/zlib/include" export CFLAGS="-fPIC -I${ROOTDIR}/build/openssl/include -I${ROOTDIR}/build/zlib/include" export LDFLAGS="-L${ROOTDIR}/build/openssl/lib -L${ROOTDIR}/build/zlib/lib" export LIBS="-lssl -lcrypto -lz" export AR=${CROSS_COMPILE}-ar export AS=${CROSS_COMPILE}-as export LD=${CROSS_COMPILE}-ld export RANLIB=${CROSS_COMPILE}-ranlib export CC=${CROSS_COMPILE}-gcc export CXX=${CROSS_COMPILE}-g++ export NM=${CROSS_COMPILE}-nm ./configure --prefix=${ROOTDIR}/build/curl --target=${CROSS_COMPILE} --host=${CROSS_COMPILE} --build=i686-linux --enable-static=libcurl.a --enable-shared=libcurl.so --enable-symbol-hiding --enable-optimize --enable-ftp --enable-http --enable-file --enable-proxy --enable-tftp --enable-smtp --enable-telnet --enable-cookies --enable-ipv6 --with-ssl --with-zlib --without-libssh2 --with-random=/dev/urandom libcurl configure配置成功之后，cd进代码目录执行make && make install命令即可
本配置使用的是android的ndk工具链gcc 4.8在配置openssl时，指定了ANDROID_NDK_ROOT的值为ndk的路径，可以参看脚本的值进行对应的设置
可以在ndk目录的build/tools目录找到make-standalone-toolchain.sh文件，执行make-standalone-toolchain.sh --help --help来查看帮助
构建自己的ndk gcc工具链，最后将生成的工具链路径加入进环境变量PATH即可
封装libcurl库
代码使用C++封装，并且使用了C++11的特性，编译时需要指定-std=c++11
头文件:
#ifndef __HTTP_REQUEST_H #define __HTTP_REQUEST_H #include <string> #include <map> #include <memory> #include <functional> #include <vector> //************************************ // Usage: // class MyResultClass // { // public: // MyResultClass() : m_request_finished(false) { } // ~MyResultClass() { } // // public: // void MyRequestResultCallback(int id, bool success, const std::string& data) // { // if (success) // { // std::ofstream outfile; // outfile.open("baidu.html", std::ios_base::binary | std::ios_base::trunc); // if (outfile.good()) outfile.write(data.c_str(), data.size()); // } // m_request_finished = true; // } // bool IsRequestFinish(void) { return m_request_finished; } // private: // bool m_request_finished; // }; // // MyResultClass mc; // HttpRequest request; // request.SetRequestUrl("http://www.baidu.com"); // request.SetResultCallback(std::bind(&MyResultClass::MyRequestResultCallback, &mc, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); // request.SetRequestHeader("User-Agent:Mozilla/4.04[en](Win95;I;Nav)"); // HANDLE hRequest = request.PerformRequest(HttpRequest::REQUEST_ASYNC); // if (hRequest) // { // while (mc.IsRequestFinish() == false) Sleep(300); // long http_code; // if (request.GetHttpCode(hRequest, &http_code)) // std::cout << "http code: " << http_code << std::endl; // std::string header; // if (request.GetReceiveHeader(hRequest, &header)) // std::cout << header << std::endl; // HttpRequest::Close(hRequest); // } // /*recommended HttpRequest::Close(hRequest) while doing async request job and dont need request handle anymore*/ //************************************ class HttpLock; #ifndef _WIN32 typedef void* HANDLE; #endif class HttpRequest { public: typedef enum { REQUEST_SYNC, REQUEST_ASYNC, }RequestType; typedef enum { REQUEST_OK, REQUEST_INVALID_OPT, REQUEST_PERFORM_ERROR, REQUEST_OPENFILE_ERROR, REQUEST_INIT_ERROR, }RequestResult; //int id, bool success, const std::string& data typedef std::function<void(int, bool, const std::string&)> ResultCallback; friend class HttpHelper; HttpRequest(); ~HttpRequest(); int SetRetryTimes(int retry_times = s_kRetryCount); int SetRequestId(int id); int SetRequestTimeout(long time_out = 0); int SetRequestUrl(const std::string& url); //************************************ // Method: SetMovedUrl // FullName: HttpRequest::SetMovedUrl // Access: public // Returns: int // Description: set http redirect follow location // Parameter: bool get_moved_url -- true means redirect http url //************************************ int SetMovedUrl(bool get_moved_url); int SetPostData(const std::string& message); int SetPostData(const void* data, unsigned int size); //************************************ // Method: SetRequestHeader // FullName: HttpRequest::SetRequestHeader // Access: public // Returns: int // Description: set http request header, for example : Range:bytes=554554- // Parameter: std::map<std::string, std::string>& // Parameter: std::string> & headers //************************************ int SetRequestHeader(std::map<std::string, std::string>& headers); int SetRequestHeader(const std::string& header); int SetRequestProxy(const std::string& proxy, long proxy_port); int SetResultCallback(ResultCallback rc); HANDLE PerformRequest(RequestType request_type); static void Close(HANDLE request_handle); bool GetHttpCode(HANDLE request_handle, long* http_code); bool GetReceiveHeader(HANDLE request_handle, std::string* header); bool GetReceiveContent(HANDLE request_handle, std::string* receive); bool GetErrorString(HANDLE request_handle, std::string* error_string); protected: class RequestHelper { public: RequestHelper(); ~RequestHelper(); friend class HttpRequest; friend class HttpHelper; int SetRetryTimes(int retry_times) { m_retry_times = retry_times; return REQUEST_OK; } int SetRequestTimeout(long time_out = 0); int SetRequestUrl(const std::string& url); int SetMovedUrl(bool get_moved_url); int SetPostData(const void* data, unsigned int size); int SetRequestHeader(const std::string& header); int SetRequestProxy(const std::string& proxy, long proxy_port); int SetResultCallback(ResultCallback rc); int Perform(); long GetHttpCode() { return m_http_code; } bool GetHeader(std::string* header); bool GetContent(std::string* receive); bool GetErrorString(std::string* error_string); bool SelfClose(void) { return m_close_self; } protected: void ReqeustResultDefault(int id, bool success, const std::string& data); private: HANDLE m_curl_handle; HANDLE m_http_headers; #ifdef _WIN32 HANDLE m_perform_thread; #else pthread_t m_perform_thread; #endif int m_retry_times; int m_id; bool m_close_self; bool m_is_running; long m_http_code; std::string m_receive_content; std::string m_receive_header; std::string m_error_string; char* m_post_data; ResultCallback m_result_callback; }; private: std::shared_ptr<RequestHelper> m_request_handle; static const int s_kRetryCount = 3; }; //************************************ // Usage: HttpDownloader // class DownCallbackClass // { // public: // DownCallbackClass() :m_down_finished(false) {} // ~DownCallbackClass() {} // public: // void DownResultCallback(int id, bool success, const std::string& data) // { // m_down_finished = true; // } // int down_callback(double total_size, double downloaded_size, void* userdata) // { // long tmp = static_cast<long>(downloaded_size / total_size * 100); // printf("\r下载进度%d", tmp); // return 0; // } // bool IsDownFinished(void) { return m_down_finished; } // private: // bool m_down_finished; // }; // HttpDownloader download; // DownCallbackClass dc; // const char* down_url = "http://dlsw.baidu.com/sw-search-sp/soft/71/10998/OfflineBaiduPlayer_151_V4.1.2.263.1432003947.exe"; // const char* down_file = "BaiduPlayer.exe"; // // download.SetDownloadUrl(down_url); // download.SetProgressCallback(std::bind(&DownCallbackClass::down_callback, &dc, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); // download.SetResultCallback(std::bind(&DownCallbackClass::DownResultCallback, &dc, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); // download.DownloadFile(down_file); // HANDLE hDownload = download.StartDownload(HttpDownloader::DOWN_ASYNC); // if (hDownload) // { // while (dc.IsDownFinished() == false) Sleep(300); // //to do download finish clean up // HttpDownloader::Close(hDownload); // } //************************************ class HttpDownloader { public: typedef enum { DOWN_SYNC, DOWN_ASYNC, }DownType; //double total_size, double downloaded_size, void* userdata typedef std::function<int(double, double, void*)> ProgressCallback; //int id, bool success, const std::string& data typedef std::function<void(int, bool, const std::string&)> ResultCallback; friend class HttpHelper; HttpDownloader(); ~HttpDownloader(); int SetRequestProxy(const std::string& proxy, long proxy_port); int SetRetryTimes(int retry_times = s_kRetryCount); int SetTimeout(long time_out = 0); int SetDownloadUrl(const std::string& url); int SetUserData(void* userdata); int SetRequestId(int id); int SetProgressCallback(ProgressCallback pc); int SetResultCallback(ResultCallback rc); int DownloadFile(const std::string& file_name, int thread_count = 5); HANDLE StartDownload(DownType down_type); static bool CancelDownload(HANDLE handle); static void Close(HANDLE handle); bool GetHttpCode(HANDLE handle, long* http_code); bool GetReceiveHeader(HANDLE handle, std::string* header); bool GetErrorString(HANDLE handle, std::string* error_string); void* GetUserData(HANDLE handle); protected: class DownloadHelper { public: typedef struct tThreadChunk { FILE* _fp; long _startidx; long _endidx; DownloadHelper* _download; }ThreadChunk; DownloadHelper(); ~DownloadHelper(); friend class HttpDownloader; friend class HttpHelper; friend ThreadChunk; void SetRetryTimes(int retry_times) { m_retry_times = retry_times; } void SetRequestId(int id) { m_id = id; } int SetTimeout(long time_out = 0); int SetRequestUrl(const std::string& url); int SetRequestProxy(const std::string& proxy, long proxy_port); void SetUserData(void *userdata) { m_userdata = userdata; } int SetProgressCallback(ProgressCallback pc); int SetResultCallback(ResultCallback rc); int SetDownloadFile(const std::string& file_name); int SetDownloadThreadCount(int thread_count); int Perform(); int GetHttpCode() { return m_http_code; } bool GetHeader(std::string* header); bool GetErrorString(std::string* error_string); bool SelfClose(void) { return m_close_self; } void* GetUserData(void) { return m_userdata; } protected: int DownloadDefaultCallback(double total_size, double downloaded_size, void* userdata); void ResultDefaultCallback(int id, bool success, const std::string& data); double GetDownloadFileSize(); int DoDownload(ThreadChunk* thread_chunk); int SplitDownloadCount(double down_size); private: #ifdef _WIN32 HANDLE m_perform_thread; #else pthread_t m_perform_thread; #endif int m_retry_times; int m_thread_count; int m_id; long m_time_out; std::string m_file_path; std::string m_url; std::string m_http_proxy; std::string m_receive_header; std::string m_error_string; bool m_close_self; bool m_multi_download; bool m_download_fail; bool m_is_running; bool m_is_cancel; void* m_userdata; long m_http_code; long m_proxy_port; double m_total_size; double m_downloaded_size; std::shared_ptr<HttpLock> m_httplock; ProgressCallback m_download_callback; ResultCallback m_result_callback; }; private: std::shared_ptr<DownloadHelper> m_request_handle; static const int s_kRetryCount = 3; static const int s_kThreadCount = 4; }; #endif /*__HTTP_REQUEST_H*/ HttpRequest.h实现文件：
//created by carbon @ 2015-05-29 /* _ooOoo_ o8888888o 88" . "88 (| -_- |) O\ = /O ___/`---'\____ .' \\| |// `. / \\||| : |||// \ / _||||| -:- |||||- \ | | \\\ - /// | | | \_| ''\---/'' | | \ .-\__ `-` ___/-. / ___`. .' /--.--\ `. . __ ."" '< `.___\_<|>_/___.' >'"". | | : `- \`.;`\ _ /`;.`/ - ` : | | \ \ `-. \_ __\ /__ _/ .-` / / ======`-.____`-.___\_____/___.-`____.-'====== `=---=' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 佛祖保佑 永无BUG */ #ifdef _WIN32 #include "stdafx.h" #else #include <pthread.h> #include <stdio.h> #include <unistd.h> #endif #include "HttpRequest.h" //HttpRequest class #include "curl/curl.h" //libcurl interface #include <list> #include <regex> #include <sstream> #ifndef _WIN32 typedef unsigned long DWORD; #define INVALID_HANDLE_VALUE (void*)0xffffffff #define TRUE 1 #define FALSE 0 #endif //#ifndef _WIN32 class HttpLock { public: #ifdef _WIN32 HttpLock() { InitializeCriticalSection(&_cs); } ~HttpLock() { DeleteCriticalSection(&_cs); } void Lock() { EnterCriticalSection(&_cs); } void UnLock() { LeaveCriticalSection(&_cs); } #else HttpLock() { pthread_mutex_init(&_lock, NULL); } ~HttpLock() { pthread_mutex_destroy(&_lock); } int Lock(){ return pthread_mutex_lock(&_lock); } int UnLock() { return pthread_mutex_unlock(&_lock); } #endif private: #ifdef _WIN32 CRITICAL_SECTION _cs; #else pthread_mutex_t _lock; #endif }; class HttpHelper { protected: HttpHelper() { curl_global_init(CURL_GLOBAL_DEFAULT); s_share_handle = curl_share_init(); curl_share_setopt(s_share_handle, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); } public: ~HttpHelper() { curl_share_cleanup(s_share_handle); curl_global_cleanup(); s_async_requests.clear(); s_async_downloads.clear(); } static HttpHelper& Instance() { static HttpHelper the_single_instance; s_id++; return the_single_instance; } static void set_share_handle(CURL* curl_handle) { curl_easy_setopt(curl_handle, CURLOPT_SHARE, s_share_handle); curl_easy_setopt(curl_handle, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 5); } static std::list< std::shared_ptr<HttpRequest::RequestHelper> > s_async_requests; static std::list< std::shared_ptr<HttpDownloader::DownloadHelper> > s_async_downloads; static int s_id; static HttpLock s_request_lock; static HttpLock s_download_lock; static CURLSH* s_share_handle; #ifdef _WIN32 static DWORD WINAPI RequestThread(LPVOID param) #else static void* RequestThread(void* param) #endif { #ifdef _WIN32 Sleep(10); #else usleep(10 * 1000); #endif std::shared_ptr<HttpRequest::RequestHelper>* request = reinterpret_cast<std::shared_ptr<HttpRequest::RequestHelper>*>(param); if (request) { (*request)->Perform(); if ((*request)->SelfClose()) { s_request_lock.Lock(); HttpHelper::s_async_requests.remove(*request); s_request_lock.UnLock(); } } #ifdef _WIN32 return 1; #else return NULL; #endif } static size_t RetriveHeaderFunction(void *ptr, size_t size, size_t nmemb, void *stream) { std::string* receive_header = reinterpret_cast<std::string*>(stream); if (receive_header && ptr) { receive_header->append(reinterpret_cast<const char*>(ptr), size * nmemb); } return nmemb * size; } static size_t RetriveContentFunction(void *ptr, size_t size, size_t nmemb, void *stream) { std::string* receive_content = reinterpret_cast<std::string*>(stream); if (receive_content && ptr) { receive_content->append(reinterpret_cast<const char*>(ptr), size * nmemb); } return nmemb * size; } #ifdef _WIN32 static DWORD WINAPI DownloadThread(LPVOID param) #else static void* DownloadThread(void* param) #endif { #ifdef _WIN32 Sleep(10); #else usleep(10 * 1000); #endif std::shared_ptr<HttpDownloader::DownloadHelper>* request = reinterpret_cast<std::shared_ptr<HttpDownloader::DownloadHelper>*>(param); if (request) { (*request)->Perform(); if ((*request)->SelfClose()) { s_download_lock.Lock(); HttpHelper::s_async_downloads.remove(*request); s_download_lock.UnLock(); } } #ifdef _WIN32 return 1; #else return NULL; #endif } static size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata) { HttpDownloader::DownloadHelper::ThreadChunk* thread_chunk = reinterpret_cast<HttpDownloader::DownloadHelper::ThreadChunk*>(userdata); if (thread_chunk->_download->m_is_cancel) { return 0; } thread_chunk->_download->m_httplock->Lock(); size_t written = 0; if (thread_chunk->_startidx <= thread_chunk->_endidx) { int real_size = size * nmemb; if (thread_chunk->_startidx + real_size > thread_chunk->_endidx) { real_size = thread_chunk->_endidx - thread_chunk->_startidx + 1; } if (fseek(thread_chunk->_fp, thread_chunk->_startidx, SEEK_SET) != 0) { perror("fseek"); } else { written = fwrite(ptr, 1, real_size, thread_chunk->_fp); thread_chunk->_startidx += written; } thread_chunk->_download->m_downloaded_size += written; } thread_chunk->_download->m_httplock->UnLock(); return written; } static int progress_callback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { HttpDownloader::DownloadHelper::ThreadChunk* thread_chunk = reinterpret_cast<HttpDownloader::DownloadHelper::ThreadChunk*>(clientp); thread_chunk->_download->m_httplock->Lock(); double total_size = thread_chunk->_download->m_total_size; double downloaded_size = thread_chunk->_download->m_downloaded_size; void* userdata = thread_chunk->_download->m_userdata; int callback_result = thread_chunk->_download->m_download_callback(total_size, downloaded_size, userdata); thread_chunk->_download->m_httplock->UnLock(); return callback_result; } #ifdef _WIN32 static DWORD WINAPI DownloadWork(LPVOID param) #else static void* DownloadWork(void* param) #endif { HttpDownloader::DownloadHelper::ThreadChunk* thread_chunk = reinterpret_cast<HttpDownloader::DownloadHelper::ThreadChunk*>(param); #ifdef _WIN32 return thread_chunk->_download->DoDownload(thread_chunk); #else return (void *)(thread_chunk->_download->DoDownload(thread_chunk)); #endif } }; std::list< std::shared_ptr<HttpRequest::RequestHelper> > HttpHelper::s_async_requests; std::list< std::shared_ptr<HttpDownloader::DownloadHelper> > HttpHelper::s_async_downloads; int HttpHelper::s_id = 0; HttpLock HttpHelper::s_request_lock; HttpLock HttpHelper::s_download_lock; CURLSH* HttpHelper::s_share_handle = nullptr; HttpRequest::HttpRequest() : m_request_handle(new HttpRequest::RequestHelper) { HttpHelper::Instance(); } HttpRequest::~HttpRequest() { } int HttpRequest::SetRetryTimes(int retry_times) { if (m_request_handle) { m_request_handle->SetRetryTimes(retry_times); return REQUEST_OK; } return REQUEST_INIT_ERROR; } int HttpRequest::SetRequestId(int id) { if (m_request_handle) { m_request_handle->m_id = id; return REQUEST_OK; } return REQUEST_INIT_ERROR; } int HttpRequest::SetRequestTimeout(long time_out) { if (m_request_handle) { if (m_request_handle->SetRequestTimeout(time_out) == CURLE_OK) { return REQUEST_OK; } else { return REQUEST_INVALID_OPT; } } return REQUEST_INIT_ERROR; } int HttpRequest::SetRequestUrl(const std::string& url) { if (m_request_handle) { if (m_request_handle->SetRequestUrl(url) == CURLE_OK) { return REQUEST_OK; } else { return REQUEST_INVALID_OPT; } } return REQUEST_INIT_ERROR; } int HttpRequest::SetMovedUrl(bool get_moved_url) { if (m_request_handle) { if (m_request_handle->SetMovedUrl(get_moved_url) == CURLE_OK) { return REQUEST_OK; } else { return REQUEST_INVALID_OPT; } } return REQUEST_INIT_ERROR; } int HttpRequest::SetPostData(const std::string& message) { return SetPostData(message.c_str(), message.size()); } int HttpRequest::SetPostData(const void* data, unsigned int size) { if (m_request_handle) { if (m_request_handle->SetPostData(data, size) == CURLE_OK) { return REQUEST_OK; } else { return REQUEST_INVALID_OPT; } } return REQUEST_INIT_ERROR; } int HttpRequest::SetRequestHeader(std::map<std::string, std::string>& headers) { if (m_request_handle) { for (auto it = headers.begin(); it != headers.end(); ++it) { std::string header = it->first; header += ": "; header += it->second; if (m_request_handle->SetRequestHeader(header) != CURLE_OK) { return REQUEST_INVALID_OPT; } } return REQUEST_OK; } return REQUEST_INIT_ERROR; } int HttpRequest::SetRequestHeader(const std::string& header) { if (m_request_handle) { if (m_request_handle->SetRequestHeader(header) == CURLE_OK) { return REQUEST_OK; } else { return REQUEST_INVALID_OPT; } } return REQUEST_INIT_ERROR; } int HttpRequest::SetRequestProxy(const std::string& proxy, long proxy_port) { if (m_request_handle) { if (m_request_handle->SetRequestProxy(proxy, proxy_port) == CURLE_OK) { return REQUEST_OK; } else { return REQUEST_INVALID_OPT; } } return REQUEST_INIT_ERROR; } int HttpRequest::SetResultCallback(ResultCallback rc) { if (m_request_handle) { m_request_handle->SetResultCallback(rc); return REQUEST_OK; } return REQUEST_INIT_ERROR; } void HttpRequest::Close(HANDLE request_handle) { std::shared_ptr<RequestHelper>* request = (reinterpret_cast<std::shared_ptr<RequestHelper> *>(request_handle)); if (request == INVALID_HANDLE_VALUE || request == nullptr) { return; } bool basync = false; HttpHelper::s_request_lock.Lock(); for (auto it = HttpHelper::s_async_requests.begin(); it != HttpHelper::s_async_requests.end(); ++it) { if ((*request) == *it) { #ifdef _WIN32 if (WaitForSingleObject((*request)->m_perform_thread, 10) == WAIT_OBJECT_0) #else if(pthread_kill((*request)->m_perform_thread, 0) != 0) #endif { HttpHelper::s_async_requests.remove(*request); } else { (*request)->m_close_self = true; } basync = true; break; } } HttpHelper::s_request_lock.UnLock(); if (basync == false) { //request->reset(); } } HANDLE HttpRequest::PerformRequest(RequestType request_type) { if (m_request_handle) { if (m_request_handle->m_is_running) { return nullptr; } if (request_type == REQUEST_SYNC) { m_request_handle->Perform(); return &m_request_handle; } else if (request_type == REQUEST_ASYNC) { HttpHelper::s_request_lock.Lock(); HttpHelper::s_async_requests.push_back(m_request_handle); std::shared_ptr<RequestHelper>& request = HttpHelper::s_async_requests.back(); #ifdef _WIN32 DWORD thread_id; HANDLE async_thread = CreateThread(NULL, 0, HttpHelper::RequestThread, &request, 0, &thread_id); request->m_perform_thread = async_thread; #else pthread_create(&(request->m_perform_thread), NULL, HttpHelper::RequestThread, &request); #endif HttpHelper::s_request_lock.UnLock(); return &request; } return nullptr; } return nullptr; } bool HttpRequest::GetHttpCode(HANDLE request_handle, long* http_code) { std::shared_ptr<RequestHelper>* request = reinterpret_cast<std::shared_ptr<RequestHelper>*>(request_handle); if (request && http_code) { *http_code = (*request)->GetHttpCode(); return true; } return false; } bool HttpRequest::GetReceiveHeader(HANDLE request_handle, std::string* header) { std::shared_ptr<RequestHelper>* request = reinterpret_cast<std::shared_ptr<RequestHelper>*>(request_handle); if (request) { return (*request)->GetHeader(header); } return false; } bool HttpRequest::GetReceiveContent(HANDLE request_handle, std::string* receive) { std::shared_ptr<RequestHelper>* request = reinterpret_cast<std::shared_ptr<RequestHelper>*>(request_handle); if (request) { return (*request)->GetContent(receive); } return false; } bool HttpRequest::GetErrorString(HANDLE request_handle, std::string* error_string) { std::shared_ptr<RequestHelper>* request = reinterpret_cast<std::shared_ptr<RequestHelper>*>(request_handle); if (request) { return (*request)->GetErrorString(error_string); } return false; } HttpRequest::RequestHelper::RequestHelper() : m_curl_handle(nullptr) #ifdef _WIN32 , m_perform_thread(nullptr) #else , m_perform_thread(-1) #endif , m_http_headers(nullptr) , m_close_self(false) , m_is_running(false) , m_retry_times(HttpRequest::s_kRetryCount) , m_http_code(0) , m_post_data(nullptr) { m_result_callback = std::bind(&RequestHelper::ReqeustResultDefault, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3); m_id = HttpHelper::s_id; m_curl_handle = curl_easy_init(); HttpHelper::set_share_handle(m_curl_handle); } HttpRequest::RequestHelper::~RequestHelper() { if (m_curl_handle) { curl_easy_cleanup(m_curl_handle); } if (m_http_headers) { curl_slist_free_all(reinterpret_cast<curl_slist*>(m_http_headers)); } if (m_post_data) { delete m_post_data; m_post_data = nullptr; } #ifdef _WIN32 if (m_perform_thread) { CloseHandle(m_perform_thread); } #endif } int HttpRequest::RequestHelper::SetRequestTimeout(long time_out) { if (m_curl_handle) { return curl_easy_setopt(m_curl_handle, CURLOPT_TIMEOUT, 0); } return CURLE_FAILED_INIT; } int HttpRequest::RequestHelper::SetRequestUrl(const std::string& url) { if (m_curl_handle) { if (url.substr(0, 5) == "https") { curl_easy_setopt(m_curl_handle, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(m_curl_handle, CURLOPT_SSL_VERIFYHOST, 0L); } return curl_easy_setopt(m_curl_handle, CURLOPT_URL, url.c_str()); } return CURLE_FAILED_INIT; } int HttpRequest::RequestHelper::SetMovedUrl(bool get_moved_url) { if (m_curl_handle) { if (get_moved_url) { curl_easy_setopt(m_curl_handle, CURLOPT_MAXREDIRS, 5); return curl_easy_setopt(m_curl_handle, CURLOPT_FOLLOWLOCATION, 1L); } else { return curl_easy_setopt(m_curl_handle, CURLOPT_FOLLOWLOCATION, 0L); } } return CURLE_FAILED_INIT; } int HttpRequest::RequestHelper::SetPostData(const void* data, unsigned int size) { if (m_curl_handle && data && size > 0) { CURLcode curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_POST, 1); if (curl_code == CURLE_OK) { if (m_post_data) { delete m_post_data; m_post_data = nullptr; } m_post_data = new char[size]; memcpy(m_post_data, data, size); curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_POSTFIELDS, m_post_data); } if (curl_code == CURLE_OK) { curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_POSTFIELDSIZE, size); } return curl_code; } return CURLE_FAILED_INIT; } int HttpRequest::RequestHelper::SetRequestHeader(const std::string& header) { if (m_curl_handle && header.empty() == false) { m_http_headers = curl_slist_append(reinterpret_cast<curl_slist*>(m_http_headers), header.c_str()); return m_http_headers ? CURLE_OK : CURLE_FAILED_INIT; } return CURLE_FAILED_INIT; } int HttpRequest::RequestHelper::SetRequestProxy(const std::string& proxy, long proxy_port) { //CURLOPT_PROXY if (m_curl_handle) { CURLcode curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_PROXYPORT, proxy_port); curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_PROXY, proxy.c_str()); return curl_code; } return CURLE_FAILED_INIT; } int HttpRequest::RequestHelper::SetResultCallback(ResultCallback rc) { m_result_callback = rc; return CURLE_OK; } void HttpRequest::RequestHelper::ReqeustResultDefault(int id, bool success, const std::string& data) { //default request callback do nothing } int HttpRequest::RequestHelper::Perform() { if (m_curl_handle) { CURLcode curl_code; if (m_http_headers) { curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_HTTPHEADER, reinterpret_cast<curl_slist*>(m_http_headers)); if (curl_code != CURLE_OK) { return curl_code; } } m_is_running = true; m_receive_header.clear(); m_receive_content.clear(); //set force http redirect SetMovedUrl(true); curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_HEADERFUNCTION, HttpHelper::RetriveHeaderFunction); curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_HEADERDATA, &m_receive_header); curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_WRITEFUNCTION, HttpHelper::RetriveContentFunction); curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_WRITEDATA, &m_receive_content); curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_NOPROGRESS, 1); curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_NOSIGNAL, 1); curl_code = curl_easy_setopt(m_curl_handle, CURLOPT_CONNECTTIMEOUT_MS, 0); curl_code = curl_easy_perform(m_curl_handle); if (curl_code == CURLE_OPERATION_TIMEDOUT) { int retry_count = m_retry_times; while (retry_count > 0) { curl_code = curl_easy_perform(m_curl_handle); if (curl_code != CURLE_OPERATION_TIMEDOUT) break; retry_count--; } } curl_easy_getinfo(m_curl_handle, CURLINFO_RESPONSE_CODE, &m_http_code); if (curl_code == CURLE_OK && m_http_code == 200) { m_result_callback(m_id, true, m_receive_content); } else { const char* err_string = curl_easy_strerror(curl_code); m_error_string = err_string; curl_code = CURLE_HTTP_POST_ERROR; m_result_callback(m_id, false, m_receive_content); } m_is_running = false; if (m_http_headers) { curl_slist_free_all(reinterpret_cast<curl_slist*>(m_http_headers)); m_http_headers = nullptr; } return curl_code; } return CURLE_FAILED_INIT; } bool HttpRequest::RequestHelper::GetHeader(std::string* header) { if (m_receive_header.empty()) return false; else if (header) *header = m_receive_header; return true; } bool HttpRequest::RequestHelper::GetContent(std::string* receive) { if (m_receive_content.empty()) return false; else if (receive) *receive = m_receive_content; return true; } bool HttpRequest::RequestHelper::GetErrorString(std::string* error_string) { if (m_error_string.empty()) return false; else if (error_string) *error_string = m_error_string; return true; } HttpDownloader::HttpDownloader() :m_request_handle(new HttpDownloader::DownloadHelper) { HttpHelper::Instance(); } HttpDownloader::~HttpDownloader() { } int HttpDownloader::SetRequestProxy(const std::string& proxy, long proxy_port) { if (m_request_handle) { if (m_request_handle->SetRequestProxy(proxy, proxy_port) == CURLE_OK) { return 0; } else { return HttpRequest::REQUEST_INVALID_OPT; } } return HttpRequest::REQUEST_INIT_ERROR; } int HttpDownloader::SetRetryTimes(int retry_times /* = s_kRetryCount */) { if (m_request_handle) { m_request_handle->SetRetryTimes(retry_times); return HttpRequest::REQUEST_OK; } return HttpRequest::REQUEST_INIT_ERROR; } int HttpDownloader::SetTimeout(long time_out /* = 0 */) { if (m_request_handle) { if (m_request_handle->SetTimeout(time_out) == CURLE_OK) { return HttpRequest::REQUEST_OK; } else { return HttpRequest::REQUEST_INVALID_OPT; } } return HttpRequest::REQUEST_INIT_ERROR; } int HttpDownloader::SetDownloadUrl(const std::string& url) { if (m_request_handle) { if (m_request_handle->SetRequestUrl(url) == CURLE_OK) { return HttpRequest::REQUEST_OK; } else { return HttpRequest::REQUEST_INVALID_OPT; } } return HttpRequest::REQUEST_INIT_ERROR; } int HttpDownloader::SetUserData(void* userdata) { if (m_request_handle) { m_request_handle->SetUserData(userdata); return HttpRequest::REQUEST_OK; } return HttpRequest::REQUEST_INIT_ERROR; } int HttpDownloader::SetRequestId(int id) { if (m_request_handle) { m_request_handle->SetRequestId(id); return HttpRequest::REQUEST_OK; } return HttpRequest::REQUEST_INIT_ERROR; } int HttpDownloader::SetProgressCallback(ProgressCallback pc) { if (m_request_handle) { m_request_handle->SetProgressCallback(pc); return HttpRequest::REQUEST_OK; } return HttpRequest::REQUEST_INIT_ERROR; } int HttpDownloader::SetResultCallback(ResultCallback rc) { if (m_request_handle) { m_request_handle->SetResultCallback(rc); return HttpRequest::REQUEST_OK; } return HttpRequest::REQUEST_INIT_ERROR; } int HttpDownloader::DownloadFile(const std::string& file_name, int thread_count /* = 5 */) { if (m_request_handle) { m_request_handle->SetDownloadFile(file_name); m_request_handle->SetDownloadThreadCount(thread_count); } return HttpRequest::REQUEST_INIT_ERROR; } HANDLE HttpDownloader::StartDownload(DownType down_type) { if (m_request_handle) { if (m_request_handle->m_is_running) { return nullptr; } if (down_type == DOWN_SYNC) { m_request_handle->Perform(); return &m_request_handle; } else if (down_type == DOWN_ASYNC) { HttpHelper::s_download_lock.Lock(); HttpHelper::s_async_downloads.push_back(m_request_handle); std::shared_ptr<DownloadHelper>& request = HttpHelper::s_async_downloads.back(); #ifdef _WIN32 DWORD thread_id; HANDLE async_thread = CreateThread(NULL, 0, HttpHelper::DownloadThread, &request, 0, &thread_id); request->m_perform_thread = async_thread; #else pthread_create(&(request->m_perform_thread), NULL, HttpHelper::DownloadThread, &request); #endif HttpHelper::s_download_lock.Lock(); return &request; } return nullptr; } return nullptr; } void HttpDownloader::Close(HANDLE handle) { std::shared_ptr<DownloadHelper>* request = (reinterpret_cast<std::shared_ptr<DownloadHelper> *>(handle)); if (request == INVALID_HANDLE_VALUE || request == nullptr) { return; } bool basync = false; HttpHelper::s_download_lock.Lock(); for (auto it = HttpHelper::s_async_downloads.begin(); it != HttpHelper::s_async_downloads.end(); ++it) { if ((*request) == *it) { #ifdef _WIN32 if (WaitForSingleObject((*request)->m_perform_thread, 10) == WAIT_OBJECT_0) #else if(pthread_kill((*request)->m_perform_thread, 0) != 0) #endif { HttpHelper::s_async_downloads.remove(*request); } else { (*request)->m_close_self = true; } basync = true; break; } } HttpHelper::s_download_lock.UnLock(); if (basync == false) { (*request)->m_is_cancel = true; //request->reset(); } } bool HttpDownloader::CancelDownload(HANDLE handle) { std::shared_ptr<DownloadHelper>* request = (reinterpret_cast<std::shared_ptr<DownloadHelper> *>(handle)); if (request == INVALID_HANDLE_VALUE || request == nullptr) { return false; } (*request)->m_is_cancel = true; return true; } bool HttpDownloader::GetHttpCode(HANDLE handle, long* http_code) { std::shared_ptr<DownloadHelper>* request = reinterpret_cast<std::shared_ptr<DownloadHelper>*>(handle); if (request && http_code) { *http_code = (*request)->GetHttpCode(); return true; } return false; } bool HttpDownloader::GetErrorString(HANDLE handle, std::string* error_string) { std::shared_ptr<DownloadHelper>* request = reinterpret_cast<std::shared_ptr<DownloadHelper>*>(handle); if (request) { return (*request)->GetErrorString(error_string); } return false; } bool HttpDownloader::GetReceiveHeader(HANDLE handle, std::string* header) { std::shared_ptr<DownloadHelper>* request = reinterpret_cast<std::shared_ptr<DownloadHelper>*>(handle); if (request) { return (*request)->GetHeader(header); } return false; } void* HttpDownloader::GetUserData(HANDLE handle) { std::shared_ptr<DownloadHelper>* request = reinterpret_cast<std::shared_ptr<DownloadHelper>*>(handle); if (request) { return (*request)->GetUserData(); } return nullptr; } HttpDownloader::DownloadHelper::DownloadHelper() #ifdef _WIN32 : m_perform_thread(nullptr) #else : m_perform_thread(-1) #endif , m_close_self(false) , m_retry_times(HttpDownloader::s_kRetryCount) , m_thread_count(HttpDownloader::s_kThreadCount) , m_http_code(0) , m_time_out(0) , m_proxy_port(0) , m_total_size(0.0) , m_downloaded_size(0.0) , m_multi_download(false) , m_download_fail(true) , m_is_running(false) , m_httplock(new HttpLock) , m_userdata(NULL) { m_download_callback = std::bind(&DownloadHelper::DownloadDefaultCallback, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3); m_result_callback = std::bind(&DownloadHelper::ResultDefaultCallback, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3); m_id = HttpHelper::s_id; } HttpDownloader::DownloadHelper::~DownloadHelper() { if (m_perform_thread) { #ifdef _WIN32 CloseHandle(m_perform_thread); m_perform_thread = nullptr; #endif } } int HttpDownloader::DownloadHelper::SetTimeout(long time_out /* = 0 */) { m_time_out = time_out; return CURLE_OK; } int HttpDownloader::DownloadHelper::SetRequestUrl(const std::string& url) { m_url = url; return CURLE_OK; } int HttpDownloader::DownloadHelper::SetRequestProxy(const std::string& proxy, long proxy_port) { m_http_proxy = proxy; m_proxy_port = proxy_port; return CURLE_OK; } int HttpDownloader::DownloadHelper::SetProgressCallback(ProgressCallback pc) { m_download_callback = pc; return CURLE_OK; } int HttpDownloader::DownloadHelper::SetResultCallback(ResultCallback rc) { m_result_callback = rc; return CURLE_OK; } int HttpDownloader::DownloadHelper::SetDownloadFile(const std::string& file_name) { m_file_path = file_name; return CURLE_OK; } int HttpDownloader::DownloadHelper::SetDownloadThreadCount(int thread_count) { m_thread_count = thread_count; return CURLE_OK; } int HttpDownloader::DownloadHelper::Perform() { m_total_size = GetDownloadFileSize(); if (m_total_size < 0) { return HttpRequest::REQUEST_PERFORM_ERROR; } std::string out_file_name = m_file_path; std::string src_file_name = out_file_name; out_file_name += ".dl"; FILE *fp = nullptr; #ifdef _WIN32 fopen_s(&fp, out_file_name.c_str(), "wb"); #else fp = fopen(out_file_name.c_str(), "wb"); #endif if (!fp) { return HttpRequest::REQUEST_OPENFILE_ERROR; } //reset enviroment m_downloaded_size = 0.0; m_download_fail = false; m_is_running = true; m_is_cancel = false; int down_code = HttpRequest::REQUEST_PERFORM_ERROR; int thread_count = SplitDownloadCount(m_total_size); m_thread_count = thread_count > m_thread_count ? m_thread_count : thread_count; //文件大小有分开下载的必要并且服务器支持多线程下载时，启用多线程下载 if (m_multi_download && m_thread_count > 1) { long gap = static_cast<long>(m_total_size) / m_thread_count; #ifdef _WIN32 std::vector<HANDLE> threads; #else std::vector<pthread_t> threads; #endif for (int i = 0; i < m_thread_count; i++) { ThreadChunk* thread_chunk = new ThreadChunk; thread_chunk->_fp = fp; thread_chunk->_download = this; if (i < m_thread_count - 1) { thread_chunk->_startidx = i * gap; thread_chunk->_endidx = thread_chunk->_startidx + gap - 1; } else { thread_chunk->_startidx = i * gap; thread_chunk->_endidx = static_cast<long>(m_total_size)-1; } #ifdef _WIN32 DWORD thread_id; HANDLE hThread = CreateThread(NULL, 0, HttpHelper::DownloadWork, thread_chunk, 0, &(thread_id)); #else pthread_t hThread; pthread_create(&hThread, NULL, HttpHelper::DownloadWork, thread_chunk); #endif threads.push_back(hThread); } #ifdef _WIN32 WaitForMultipleObjects(threads.size(), &threads[0], TRUE, INFINITE); for (HANDLE handle : threads) { CloseHandle(handle); } #else for(pthread_t thread : threads) { pthread_join(thread, NULL); } #endif } else { ThreadChunk* thread_chunk = new ThreadChunk; thread_chunk->_fp = fp; thread_chunk->_download = this; thread_chunk->_startidx = 0; thread_chunk->_endidx = static_cast<long>(m_total_size)-1; down_code = DoDownload(thread_chunk); } if (m_download_fail == false) { fclose(fp); #ifdef _WIN32 MoveFileExA(out_file_name.c_str(), src_file_name.c_str(), MOVEFILE_REPLACE_EXISTING); #else unlink(src_file_name.c_str()); rename(out_file_name.c_str(), src_file_name.c_str()); #endif } else { #ifdef _WIN32 DeleteFileA(out_file_name.c_str()); #else unlink(out_file_name.c_str()); #endif } m_result_callback(m_id, m_download_fail ? false : true, ""); m_is_running = false; return down_code; } bool HttpDownloader::DownloadHelper::GetHeader(std::string* header) { if (m_receive_header.empty()) return false; else if (header) *header = m_receive_header; return true; } bool HttpDownloader::DownloadHelper::GetErrorString(std::string* error_string) { if (m_error_string.empty()) return false; else if (error_string) *error_string = m_error_string; return true; } int HttpDownloader::DownloadHelper::DownloadDefaultCallback(double total_size, double downloaded_size, void* userdata) { return static_cast<int>(downloaded_size * 100 / total_size); } void HttpDownloader::DownloadHelper::ResultDefaultCallback(int id, bool success, const std::string& data) { } double HttpDownloader::DownloadHelper::GetDownloadFileSize() { if (m_url.empty()) { return -1.0; } else { double down_file_length = -1.0; CURL *handle = curl_easy_init(); HttpHelper::set_share_handle(handle); if (handle) { curl_easy_setopt(handle, CURLOPT_URL, m_url.c_str()); curl_easy_setopt(handle, CURLOPT_HEADER, 1); curl_easy_setopt(handle, CURLOPT_NOBODY, 1); curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 5); curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, HttpHelper::RetriveHeaderFunction); curl_easy_setopt(handle, CURLOPT_HEADERDATA, &m_receive_header); curl_easy_setopt(handle, CURLOPT_RANGE, "2-"); CURLcode curl_code = curl_easy_perform(handle); if (curl_code == CURLE_OPERATION_TIMEDOUT) { int retry_count = m_retry_times; while (retry_count > 0) { curl_code = curl_easy_perform(handle); if (curl_code != CURLE_OPERATION_TIMEDOUT) break; retry_count--; } } curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &m_http_code); if (curl_code == CURLE_OK) { curl_easy_getinfo(handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &down_file_length); //匹配"Content-Range: bytes 2-1449/26620" 则证明支持多线程下载 std::regex pattern("CONTENT-RANGE\\s*:\\s*\\w+\\s*(\\d+)-(\\d*)/(\\d+)", std::regex::icase); m_multi_download = std::regex_search(m_receive_header, pattern); } else { const char* err_string = curl_easy_strerror(curl_code); m_error_string = err_string; } curl_easy_cleanup(handle); } return down_file_length; } } int HttpDownloader::DownloadHelper::DoDownload(ThreadChunk* thread_chunk) { CURL* curl_handle = curl_easy_init(); HttpHelper::set_share_handle(curl_handle); if (thread_chunk->_download->m_url.substr(0, 5) == "https") { curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L); } curl_easy_setopt(curl_handle, CURLOPT_URL, thread_chunk->_download->m_url.c_str()); const char* user_agent = ("Mozilla/5.0 (Windows NT 5.1; rv:5.0) Gecko/20100101 Firefox/5.0"); curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent); curl_easy_setopt(curl_handle, CURLOPT_MAXREDIRS, 5L); curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl_handle, CURLOPT_POST, 0L); curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT_MS, 0L); curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, thread_chunk->_download->m_time_out); //0 means block always curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, HttpHelper::write_callback); curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, thread_chunk); curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(curl_handle, CURLOPT_XFERINFOFUNCTION, HttpHelper::progress_callback); curl_easy_setopt(curl_handle, CURLOPT_XFERINFODATA, thread_chunk); curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(curl_handle, CURLOPT_LOW_SPEED_TIME, 5L); std::string down_range; std::ostringstream ostr; ostr << thread_chunk->_startidx << "-" << thread_chunk->_endidx; down_range = ostr.str(); curl_easy_setopt(curl_handle, CURLOPT_RANGE, down_range.c_str()); CURLcode curl_code = curl_easy_perform(curl_handle); if (curl_code == CURLE_OPERATION_TIMEDOUT) { int retry_count = m_retry_times; while (retry_count > 0) { curl_code = curl_easy_perform(curl_handle); if (curl_code != CURLE_OPERATION_TIMEDOUT) break; retry_count--; } } long http_code; curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code); if (curl_code == CURLE_OK && (http_code >= 200 && http_code <= 300)) { m_http_code = http_code; } else { const char* err_string = curl_easy_strerror(curl_code); m_error_string = err_string; thread_chunk->_download->m_download_fail = true; m_http_code = http_code; } curl_easy_cleanup(curl_handle); delete thread_chunk; return curl_code; } int HttpDownloader::DownloadHelper::SplitDownloadCount(double down_size) { const double size_2mb = 2.0 * 1024 * 1024; const double size_10mb = 10.0 * 1024 * 1024; const double size_50mb = 50.0 * 1024 * 1024; if (down_size <= size_2mb) { return 1; } else if (down_size > size_2mb && down_size <= size_10mb) { return static_cast<int>(down_size / (size_2mb)); } else if (down_size > size_10mb && down_size <= size_50mb) { return HttpDownloader::s_kThreadCount + 1; } else { int down_count = static_cast<int>(down_size / size_10mb); return down_count > 10 ? 10 : down_count; } return 1; } HttpRequest.cpp使用libcurl库
demo使用封装的库来模拟请求数据和下载文件。
例子很简单，直接看代码：
// http_request.cpp : 定义控制台应用程序的入口点。 // #include "HttpRequest.h" #include <iostream> #include <string> #include <fstream> #include <functional> class DownCallbackClass { public: DownCallbackClass() :m_down_finished(false) {} ~DownCallbackClass() {} public: void DownResultCallback(int id, bool success, const std::string& data) { m_down_finished = true; } int down_callback(double total_size, double downloaded_size, void* userdata) { long tmp = static_cast<long>(downloaded_size / total_size * 100); printf("\r下载进度%d", tmp); return 0; } bool IsDownFinished(void) { return m_down_finished; } private: bool m_down_finished; }; class MyResultClass { public: MyResultClass() : m_request_finished(false) { } ~MyResultClass() { } public: void MyRequestResultCallback(int id, bool success, const std::string& data) { if (success) { std::ofstream outfile; outfile.open("baidu.html", std::ios_base::binary | std::ios_base::trunc); if (outfile.good()) outfile.write(data.c_str(), data.size()); } m_request_finished = true; } bool IsRequestFinish(void) { return m_request_finished; } private: bool m_request_finished; }; int _tmain(int argc, _TCHAR* argv[]) { MyResultClass mc; HttpRequest request; request.SetRequestUrl("http://www.baidu.com"); request.SetResultCallback(std::bind(&MyResultClass::MyRequestResultCallback, &mc, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); request.SetRequestHeader("User-Agent:Mozilla/4.04[en](Win95;I;Nav)"); HANDLE hRequest = request.PerformRequest(HttpRequest::REQUEST_ASYNC); if (hRequest) { while (mc.IsRequestFinish() == false) Sleep(300); long http_code; if (request.GetHttpCode(hRequest, &http_code)) std::cout << "http code: " << http_code << std::endl; std::string header; if (request.GetReceiveHeader(hRequest, &header)) { std::cout << header << std::endl; } HttpRequest::Close(hRequest); } HttpDownloader download; DownCallbackClass dc; const char* down_url = "http://dlsw.baidu.com/sw-search-sp/soft/71/10998/OfflineBaiduPlayer_151_V4.1.2.263.1432003947.exe"; const char* down_file = "BaiduPlayer.exe"; download.SetDownloadUrl(down_url); download.SetProgressCallback(std::bind(&DownCallbackClass::down_callback, &dc, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); download.SetResultCallback(std::bind(&DownCallbackClass::DownResultCallback, &dc, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); download.DownloadFile(down_file); HANDLE hDownload = download.StartDownload(HttpDownloader::DOWN_ASYNC); if (hDownload) { while (dc.IsDownFinished() == false) { Sleep(300); } //to do download finish clean up HttpDownloader::Close(hDownload); } return 0; }来自：http://www.cnblogs.com/jojodru/p/4551201.html