[cmake-developers] [PATCH] cmFileCommand: Download continuation support
Titov Denis
kernelmd at yandex.ru
Tue Aug 16 07:11:48 EDT 2016
This patch introduces several options for the file(DOWNLOAD ...) command, which
would be useful in case of dealing with big files or unstable network
connections.
The added options are:
RETRY_COUNT <int> -- sets maximal amount of download restarts, default value: 1
RETRY_DELAY <real> -- sets delay before restarting download (in seconds),
default value: 0.0
RETRY_MAX_TIME <real> -- sets maximal time spent in downloading file
(in seconds), default value: infinity
RETRY_CONTINUE -- if set, makes cmake try to continue downloading of the
existing chunk, instead of discarding it and starting all over. This option is
not set by default
Notes:
The RETRY_CONTINUE option requires server-side support of http partial get
(content-range header).
Unfortunately, I haven't been able to properly test the RETRY_CONTINUE option,
as I didn't have access to the appropriate server. Any help in this area is
encouraged.
---
Source/cmFileCommand.cxx | 287 +++++++++++++++++++++++++++++++++--------------
1 file changed, 204 insertions(+), 83 deletions(-)
diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx
index 835b118..aa7782e 100644
--- a/Source/cmFileCommand.cxx
+++ b/Source/cmFileCommand.cxx
@@ -34,6 +34,16 @@
// include sys/stat.h after sys/types.h
#include <sys/stat.h>
+#include <float.h>
+#include <time.h>
+
+// For crossplatform_sleep().
+#if defined(_WIN32) && !defined(__CYGWIN__)
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
#include <cm_auto_ptr.hxx>
#include <cmsys/Directory.hxx>
#include <cmsys/Encoding.hxx>
@@ -68,6 +78,15 @@ static mode_t mode_setuid = S_ISUID;
static mode_t mode_setgid = S_ISGID;
#endif
+void crossplatform_sleep(int delay_s)
+{
+#if defined(_WIN32) && !defined(__CYGWIN__)
+ Sleep(delay_s);
+#else
+ sleep(delay_s * 1000);
+#endif
+}
+
#if defined(_WIN32) && defined(CMAKE_ENCODING_UTF8)
// libcurl doesn't support file:// urls for unicode filenames on Windows.
// Convert string from UTF-8 to ACP if this is a file:// URL.
@@ -2481,6 +2500,11 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
std::string hashMatchMSG;
CM_AUTO_PTR<cmCryptoHash> hash;
bool showProgress = false;
+ int retryMaxCount = 1;
+ double retryDelayS = 0.0;
+ double retryMaxTimeS = DBL_MAX;
+ bool retryContinue = false;
+ cmsys::ofstream fout;
while (i != args.end()) {
if (*i == "TIMEOUT") {
@@ -2564,7 +2588,34 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
return false;
}
hashMatchMSG = algo + " hash";
+ } else if (*i == "RETRY_COUNT") {
+ ++i;
+ if (i != args.end()) {
+ retryMaxCount = atoi(i->c_str());
+ } else {
+ this->SetError("DOWNLOAD missing count for RETRY_COUNT");
+ return false;
+ }
+ } else if (*i == "RETRY_DELAY") {
+ ++i;
+ if (i != args.end()) {
+ retryDelayS = atof(i->c_str());
+ } else {
+ this->SetError("DOWNLOAD missing time for RETRY_DELAY");
+ return false;
+ }
+ } else if (*i == "RETRY_MAX_TIME") {
+ ++i;
+ if (i != args.end()) {
+ retryMaxTimeS = atof(i->c_str());
+ } else {
+ this->SetError("DOWNLOAD missing time for RETRY_MAX_TIME");
+ return false;
+ }
+ } else if (*i == "RETRY_CONTINUE") {
+ retryContinue = true;
}
+
++i;
}
// If file exists already, and caller specified an expected md5 or sha,
@@ -2599,110 +2650,171 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
return false;
}
- cmsys::ofstream fout(file.c_str(), std::ios::binary);
- if (!fout) {
- this->SetError("DOWNLOAD cannot open file for write.");
- return false;
- }
-
#if defined(_WIN32) && defined(CMAKE_ENCODING_UTF8)
url = fix_file_url_windows(url);
#endif
+ cmFileCommandVectorOfChar chunkDebug;
+
::CURL* curl;
::curl_global_init(CURL_GLOBAL_DEFAULT);
- curl = ::curl_easy_init();
- if (!curl) {
- this->SetError("DOWNLOAD error initializing curl.");
- return false;
- }
- cURLEasyGuard g_curl(curl);
- ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
- check_curl_result(res, "DOWNLOAD cannot set url: ");
+ ::CURLcode res;
+ int tries = 0;
+ double elapsed = 0.0;
+ time_t start, end;
+ while (tries < retryMaxCount && elapsed <= retryMaxTimeS) {
+ ++tries;
+ time(&start);
+
+ curl = ::curl_easy_init();
+ if (!curl) {
+ this->SetError("DOWNLOAD error initializing curl.");
+ ::curl_global_cleanup();
+ return false;
+ }
- // enable HTTP ERROR parsing
- res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
- check_curl_result(res, "DOWNLOAD cannot set http failure option: ");
+ if (cmSystemTools::FileExists(file.c_str())) { // Something was downloaded.
+ // Check hash.
+ if (hash.get()) {
+ std::string actualHash = hash->HashFile(file);
+ if (actualHash == expectedHash) { // File is complete, exit.
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return true;
+ }
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/" LIBCURL_VERSION);
- check_curl_result(res, "DOWNLOAD cannot set user agent option: ");
+ if (retryContinue == false) { // Discard downloaded chunk.
+ fout.open(file.c_str(), std::ios::binary | std::ios::trunc);
+ if (!fout.good()) {
+ this->SetError("Cannot open file for writing");
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
+ } else { // Try to continue.
+ fout.open(file.c_str(), std::ios::binary | std::ios::app);
+ if (!fout.good()) {
+ this->SetError("Cannot open file for writing");
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
+ curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, fout.tellp());
+ }
+ } else { // Create new file.
+ fout.open(file.c_str(), std::ios::binary);
+ if (!fout.good()) {
+ this->SetError("Cannot open file for writing");
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cmWriteToFileCallback);
- check_curl_result(res, "DOWNLOAD cannot set write function: ");
+ cURLEasyGuard g_curl(curl);
+ res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+ check_curl_result(res, "DOWNLOAD cannot set url: ");
- res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
- cmFileCommandCurlDebugCallback);
- check_curl_result(res, "DOWNLOAD cannot set debug function: ");
+ // enable HTTP ERROR parsing
+ res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+ check_curl_result(res, "DOWNLOAD cannot set http failure option: ");
- // check to see if TLS verification is requested
- if (tls_verify) {
- res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
- check_curl_result(res, "Unable to set TLS/SSL Verify on: ");
- } else {
- res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
- check_curl_result(res, "Unable to set TLS/SSL Verify off: ");
- }
- // check to see if a CAINFO file has been specified
- // command arg comes first
- std::string const& cainfo_err = cmCurlSetCAInfo(curl, cainfo);
- if (!cainfo_err.empty()) {
- this->SetError(cainfo_err);
- return false;
- }
+ res = ::curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/" LIBCURL_VERSION);
+ check_curl_result(res, "DOWNLOAD cannot set user agent option: ");
- cmFileCommandVectorOfChar chunkDebug;
+ res =
+ ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cmWriteToFileCallback);
+ check_curl_result(res, "DOWNLOAD cannot set write function: ");
- res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&fout);
- check_curl_result(res, "DOWNLOAD cannot set write data: ");
+ res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
+ cmFileCommandCurlDebugCallback);
+ check_curl_result(res, "DOWNLOAD cannot set debug function: ");
- res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void*)&chunkDebug);
- check_curl_result(res, "DOWNLOAD cannot set debug data: ");
+ // check to see if TLS verification is requested
+ if (tls_verify) {
+ res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
+ check_curl_result(res, "Unable to set TLS/SSL Verify on: ");
+ } else {
+ res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+ check_curl_result(res, "Unable to set TLS/SSL Verify off: ");
+ }
+ // check to see if a CAINFO file has been specified
+ // command arg comes first
+ std::string const& cainfo_err = cmCurlSetCAInfo(curl, cainfo);
+ if (!cainfo_err.empty()) {
+ this->SetError(cainfo_err);
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
- check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: ");
+ res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&fout);
+ check_curl_result(res, "DOWNLOAD cannot set write data: ");
- if (!logVar.empty()) {
- res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
- check_curl_result(res, "DOWNLOAD cannot set verbose: ");
- }
+ res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void*)&chunkDebug);
+ check_curl_result(res, "DOWNLOAD cannot set debug data: ");
- if (timeout > 0) {
- res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
- check_curl_result(res, "DOWNLOAD cannot set timeout: ");
- }
+ res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: ");
- if (inactivity_timeout > 0) {
- // Give up if there is no progress for a long time.
- ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1);
- ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, inactivity_timeout);
- }
+ if (!logVar.empty()) {
+ res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
+ check_curl_result(res, "DOWNLOAD cannot set verbose: ");
+ }
- // Need the progress helper's scope to last through the duration of
- // the curl_easy_perform call... so this object is declared at function
- // scope intentionally, rather than inside the "if(showProgress)"
- // block...
- //
- cURLProgressHelper helper(this, "download");
+ if (timeout > 0) {
+ res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
+ check_curl_result(res, "DOWNLOAD cannot set timeout: ");
+ }
- if (showProgress) {
- res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
- check_curl_result(res, "DOWNLOAD cannot set noprogress value: ");
+ if (inactivity_timeout > 0) {
+ // Give up if there is no progress for a long time.
+ ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1);
+ ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, inactivity_timeout);
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION,
- cmFileDownloadProgressCallback);
- check_curl_result(res, "DOWNLOAD cannot set progress function: ");
+ // Need the progress helper's scope to last through the duration of
+ // the curl_easy_perform call... so this object is declared at loop
+ // scope intentionally, rather than inside the "if(showProgress)"
+ // block...
+ //
+ cURLProgressHelper helper(this, "download");
- res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSDATA,
- reinterpret_cast<void*>(&helper));
- check_curl_result(res, "DOWNLOAD cannot set progress data: ");
- }
+ if (showProgress) {
+ res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+ check_curl_result(res, "DOWNLOAD cannot set noprogress value: ");
- res = ::curl_easy_perform(curl);
+ res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION,
+ cmFileDownloadProgressCallback);
+ check_curl_result(res, "DOWNLOAD cannot set progress function: ");
- /* always cleanup */
- g_curl.release();
- ::curl_easy_cleanup(curl);
+ res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSDATA,
+ reinterpret_cast<void*>(&helper));
+ check_curl_result(res, "DOWNLOAD cannot set progress data: ");
+ }
+
+ res = ::curl_easy_perform(curl);
+
+ /* always cleanup */
+ g_curl.release();
+ ::curl_easy_cleanup(curl);
+ fout.flush();
+ fout.close();
+
+ // Download finished successfuly, exit the loop.
+ if (res == ::CURLE_OK) {
+ break;
+ // Server doesn't support content ranges...
+ } else if (retryContinue == true && res == ::CURLE_RANGE_ERROR) {
+ retryContinue = false; // Disable download continuation.
+ }
+
+ crossplatform_sleep(retryDelayS);
+ time(&end);
+ elapsed += difftime(end, start);
+ }
if (!statusVar.empty()) {
std::ostringstream result;
@@ -2712,10 +2824,19 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
::curl_global_cleanup();
- // Explicitly flush/close so we can measure the md5 accurately.
- //
- fout.flush();
- fout.close();
+ if (res != ::CURLE_OK) {
+ std::ostringstream oss;
+ // Failed by exhausting attempts
+ if (retryMaxCount != 1 && tries == retryMaxCount) {
+ oss << "Download failed after " << tries << " attempts. ";
+ }
+ // Failed by exhausting maximal time.
+ if (retryMaxTimeS < DBL_MAX && elapsed >= retryMaxTimeS) {
+ oss << "Download failed: time exhausted, " << elapsed << "s. spent. ";
+ }
+ oss << "Last CURL error: " << curl_easy_strerror(res);
+ return false;
+ }
// Verify MD5 sum if requested:
//
--
2.9.2
More information about the cmake-developers
mailing list