[cmake-developers] Subject: [PATCH v2] cmFileCommand: Download continuation support
Titov Denis
kernelmd at yandex.ru
Thu Aug 18 13:47:48 EDT 2016
This patch introduces several options for the file(DOWNLOAD ...) command, which
would be useful in case of dealing with big files or unstable network
connections.
The added options are:
RETRY_COUNT <int> -- sets maximal amount of download restarts, default value: 1
RETRY_DELAY <real> -- sets delay before restarting download (in seconds),
default value: 0.0
RETRY_MAX_TIME <real> -- sets maximal time spent in downloading file
(in seconds), default value: infinity
RETRY_CONTINUE -- if set, makes cmake try to continue downloading of the
existing chunk, instead of discarding it and starting all over. This option is
not set by default
Notes:
The RETRY_CONTINUE option requires server-side support of http partial get
(content-range header).
Unfortunately, I haven't been able to properly test the RETRY_CONTINUE option,
as I didn't have access to the appropriate server. Any help in this area is
encouraged.
---
Help/command/file.rst | 17 +++
Source/cmFileCommand.cxx | 271 ++++++++++++++++++++++++++++++++---------------
2 files changed, 205 insertions(+), 83 deletions(-)
diff --git a/Help/command/file.rst b/Help/command/file.rst
index 256d16d..f1095b7 100644
--- a/Help/command/file.rst
+++ b/Help/command/file.rst
@@ -240,6 +240,23 @@ Additional options to ``DOWNLOAD`` are:
``TLS_CAINFO <file>``
Specify a custom Certificate Authority file for ``https://`` URLs.
+``RETRY_COUNT <count>``
+ Set maximal amount of download restarts, default value: 1
+
+``RETRY_DELAY <seconds>``
+ Set delay before restarting download, default value: 0.0
+
+``RETRY_MAX_TIME <seconds>``
+ Set maximal time spent in downloading file, default value: infinity
+
+``RETRY_CONTINUE``
+ If set, makes cmake try to continue downloading of the existing chunk,
+ instead of discarding it and starting all over. This option is not set
+ by default.
+
+ Note that this option requires server-side support of http partial get
+ (content-range header).
+
For ``https://`` URLs CMake must be built with OpenSSL support. ``TLS/SSL``
certificates are not checked by default. Set ``TLS_VERIFY`` to ``ON`` to
check certificates and/or use ``EXPECTED_HASH`` to verify downloaded content.
diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx
index 835b118..bbe8839 100644
--- a/Source/cmFileCommand.cxx
+++ b/Source/cmFileCommand.cxx
@@ -34,6 +34,9 @@
// include sys/stat.h after sys/types.h
#include <sys/stat.h>
+#include <float.h>
+#include <time.h>
+
#include <cm_auto_ptr.hxx>
#include <cmsys/Directory.hxx>
#include <cmsys/Encoding.hxx>
@@ -2481,6 +2484,11 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
std::string hashMatchMSG;
CM_AUTO_PTR<cmCryptoHash> hash;
bool showProgress = false;
+ int retryMaxCount = 1;
+ double retryDelayS = 0.0;
+ double retryMaxTimeS = DBL_MAX;
+ bool retryContinue = false;
+ cmsys::ofstream fout;
while (i != args.end()) {
if (*i == "TIMEOUT") {
@@ -2564,7 +2572,34 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
return false;
}
hashMatchMSG = algo + " hash";
+ } else if (*i == "RETRY_COUNT") {
+ ++i;
+ if (i != args.end()) {
+ retryMaxCount = atoi(i->c_str());
+ } else {
+ this->SetError("DOWNLOAD missing count for RETRY_COUNT");
+ return false;
+ }
+ } else if (*i == "RETRY_DELAY") {
+ ++i;
+ if (i != args.end()) {
+ retryDelayS = atof(i->c_str());
+ } else {
+ this->SetError("DOWNLOAD missing time for RETRY_DELAY");
+ return false;
+ }
+ } else if (*i == "RETRY_MAX_TIME") {
+ ++i;
+ if (i != args.end()) {
+ retryMaxTimeS = atof(i->c_str());
+ } else {
+ this->SetError("DOWNLOAD missing time for RETRY_MAX_TIME");
+ return false;
+ }
+ } else if (*i == "RETRY_CONTINUE") {
+ retryContinue = true;
}
+
++i;
}
// If file exists already, and caller specified an expected md5 or sha,
@@ -2599,110 +2634,171 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
return false;
}
- cmsys::ofstream fout(file.c_str(), std::ios::binary);
- if (!fout) {
- this->SetError("DOWNLOAD cannot open file for write.");
- return false;
- }
-
#if defined(_WIN32) && defined(CMAKE_ENCODING_UTF8)
url = fix_file_url_windows(url);
#endif
+ cmFileCommandVectorOfChar chunkDebug;
+
::CURL* curl;
::curl_global_init(CURL_GLOBAL_DEFAULT);
- curl = ::curl_easy_init();
- if (!curl) {
- this->SetError("DOWNLOAD error initializing curl.");
- return false;
- }
- cURLEasyGuard g_curl(curl);
- ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
- check_curl_result(res, "DOWNLOAD cannot set url: ");
+ ::CURLcode res;
+ int tries = 0;
+ double elapsed = 0.0;
+ time_t start, end;
+ while (tries < retryMaxCount && elapsed <= retryMaxTimeS) {
+ ++tries;
+ time(&start);
+
+ curl = ::curl_easy_init();
+ if (!curl) {
+ this->SetError("DOWNLOAD error initializing curl.");
+ ::curl_global_cleanup();
+ return false;
+ }
- // enable HTTP ERROR parsing
- res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
- check_curl_result(res, "DOWNLOAD cannot set http failure option: ");
+ if (cmSystemTools::FileExists(file.c_str())) { // Something was downloaded.
+ // Check hash.
+ if (hash.get()) {
+ std::string actualHash = hash->HashFile(file);
+ if (actualHash == expectedHash) { // File is complete, exit.
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return true;
+ }
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/" LIBCURL_VERSION);
- check_curl_result(res, "DOWNLOAD cannot set user agent option: ");
+ if (retryContinue == false) { // Discard downloaded chunk.
+ fout.open(file.c_str(), std::ios::binary | std::ios::trunc);
+ if (!fout.good()) {
+ this->SetError("Cannot open file for writing");
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
+ } else { // Try to continue.
+ fout.open(file.c_str(), std::ios::binary | std::ios::app);
+ if (!fout.good()) {
+ this->SetError("Cannot open file for writing");
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
+ curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, fout.tellp());
+ }
+ } else { // Create new file.
+ fout.open(file.c_str(), std::ios::binary);
+ if (!fout.good()) {
+ this->SetError("Cannot open file for writing");
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cmWriteToFileCallback);
- check_curl_result(res, "DOWNLOAD cannot set write function: ");
+ cURLEasyGuard g_curl(curl);
+ res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+ check_curl_result(res, "DOWNLOAD cannot set url: ");
- res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
- cmFileCommandCurlDebugCallback);
- check_curl_result(res, "DOWNLOAD cannot set debug function: ");
+ // enable HTTP ERROR parsing
+ res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+ check_curl_result(res, "DOWNLOAD cannot set http failure option: ");
- // check to see if TLS verification is requested
- if (tls_verify) {
- res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
- check_curl_result(res, "Unable to set TLS/SSL Verify on: ");
- } else {
- res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
- check_curl_result(res, "Unable to set TLS/SSL Verify off: ");
- }
- // check to see if a CAINFO file has been specified
- // command arg comes first
- std::string const& cainfo_err = cmCurlSetCAInfo(curl, cainfo);
- if (!cainfo_err.empty()) {
- this->SetError(cainfo_err);
- return false;
- }
+ res = ::curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/" LIBCURL_VERSION);
+ check_curl_result(res, "DOWNLOAD cannot set user agent option: ");
- cmFileCommandVectorOfChar chunkDebug;
+ res =
+ ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cmWriteToFileCallback);
+ check_curl_result(res, "DOWNLOAD cannot set write function: ");
- res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&fout);
- check_curl_result(res, "DOWNLOAD cannot set write data: ");
+ res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
+ cmFileCommandCurlDebugCallback);
+ check_curl_result(res, "DOWNLOAD cannot set debug function: ");
- res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void*)&chunkDebug);
- check_curl_result(res, "DOWNLOAD cannot set debug data: ");
+ // check to see if TLS verification is requested
+ if (tls_verify) {
+ res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
+ check_curl_result(res, "Unable to set TLS/SSL Verify on: ");
+ } else {
+ res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+ check_curl_result(res, "Unable to set TLS/SSL Verify off: ");
+ }
+ // check to see if a CAINFO file has been specified
+ // command arg comes first
+ std::string const& cainfo_err = cmCurlSetCAInfo(curl, cainfo);
+ if (!cainfo_err.empty()) {
+ this->SetError(cainfo_err);
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
- check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: ");
+ res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&fout);
+ check_curl_result(res, "DOWNLOAD cannot set write data: ");
- if (!logVar.empty()) {
- res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
- check_curl_result(res, "DOWNLOAD cannot set verbose: ");
- }
+ res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void*)&chunkDebug);
+ check_curl_result(res, "DOWNLOAD cannot set debug data: ");
- if (timeout > 0) {
- res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
- check_curl_result(res, "DOWNLOAD cannot set timeout: ");
- }
+ res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: ");
- if (inactivity_timeout > 0) {
- // Give up if there is no progress for a long time.
- ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1);
- ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, inactivity_timeout);
- }
+ if (!logVar.empty()) {
+ res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
+ check_curl_result(res, "DOWNLOAD cannot set verbose: ");
+ }
- // Need the progress helper's scope to last through the duration of
- // the curl_easy_perform call... so this object is declared at function
- // scope intentionally, rather than inside the "if(showProgress)"
- // block...
- //
- cURLProgressHelper helper(this, "download");
+ if (timeout > 0) {
+ res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
+ check_curl_result(res, "DOWNLOAD cannot set timeout: ");
+ }
- if (showProgress) {
- res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
- check_curl_result(res, "DOWNLOAD cannot set noprogress value: ");
+ if (inactivity_timeout > 0) {
+ // Give up if there is no progress for a long time.
+ ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1);
+ ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, inactivity_timeout);
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION,
- cmFileDownloadProgressCallback);
- check_curl_result(res, "DOWNLOAD cannot set progress function: ");
+ // Need the progress helper's scope to last through the duration of
+ // the curl_easy_perform call... so this object is declared at loop
+ // scope intentionally, rather than inside the "if(showProgress)"
+ // block...
+ //
+ cURLProgressHelper helper(this, "download");
- res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSDATA,
- reinterpret_cast<void*>(&helper));
- check_curl_result(res, "DOWNLOAD cannot set progress data: ");
- }
+ if (showProgress) {
+ res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+ check_curl_result(res, "DOWNLOAD cannot set noprogress value: ");
- res = ::curl_easy_perform(curl);
+ res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION,
+ cmFileDownloadProgressCallback);
+ check_curl_result(res, "DOWNLOAD cannot set progress function: ");
- /* always cleanup */
- g_curl.release();
- ::curl_easy_cleanup(curl);
+ res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSDATA,
+ reinterpret_cast<void*>(&helper));
+ check_curl_result(res, "DOWNLOAD cannot set progress data: ");
+ }
+
+ res = ::curl_easy_perform(curl);
+
+ /* always cleanup */
+ g_curl.release();
+ ::curl_easy_cleanup(curl);
+ fout.flush();
+ fout.close();
+
+ // Download finished successfuly, exit the loop.
+ if (res == ::CURLE_OK) {
+ break;
+ // Server doesn't support content ranges...
+ } else if (retryContinue == true && res == ::CURLE_RANGE_ERROR) {
+ retryContinue = false; // Disable download continuation.
+ }
+
+ cmUtils::Delay(retryDelayS * 1000);
+ time(&end);
+ elapsed += difftime(end, start);
+ }
if (!statusVar.empty()) {
std::ostringstream result;
@@ -2712,10 +2808,19 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
::curl_global_cleanup();
- // Explicitly flush/close so we can measure the md5 accurately.
- //
- fout.flush();
- fout.close();
+ if (res != ::CURLE_OK) {
+ std::ostringstream oss;
+ // Failed by exhausting attempts
+ if (retryMaxCount != 1 && tries == retryMaxCount) {
+ oss << "Download failed after " << tries << " attempts. ";
+ }
+ // Failed by exhausting maximal time.
+ if (retryMaxTimeS < DBL_MAX && elapsed >= retryMaxTimeS) {
+ oss << "Download failed: time exhausted, " << elapsed << "s. spent. ";
+ }
+ oss << "Last CURL error: " << curl_easy_strerror(res);
+ return false;
+ }
// Verify MD5 sum if requested:
//
--
2.9.2
More information about the cmake-developers
mailing list