* [PATCH 0/2] cve-check-tool progress reporting
@ 2016-09-26 15:42 André Draszik
2016-09-26 15:42 ` [PATCH 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh André Draszik
` (2 more replies)
0 siblings, 3 replies; 8+ messages in thread
From: André Draszik @ 2016-09-26 15:42 UTC (permalink / raw)
To: openembedded-core
These patches implement progress reporting for cve-check-tool when
it is downloading the CVE database.
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh 2016-09-26 15:42 [PATCH 0/2] cve-check-tool progress reporting André Draszik @ 2016-09-26 15:42 ` André Draszik 2016-09-26 15:42 ` [PATCH 2/2] cve-check-tool: report progress when downloading CVE database André Draszik 2016-09-28 12:05 ` [PATCH v2 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh André Draszik 2 siblings, 0 replies; 8+ messages in thread From: André Draszik @ 2016-09-26 15:42 UTC (permalink / raw) To: openembedded-core This will allow us to easily incorporate progress support via bb.process.run() Signed-off-by: André Draszik <git@andred.net> --- .../cve-check-tool/cve-check-tool_5.6.4.bb | 45 +++++++++------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb b/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb index 1165559..5bb22d1 100644 --- a/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb +++ b/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb @@ -22,35 +22,28 @@ inherit pkgconfig autotools EXTRA_OECONF = "--disable-coverage" CFLAGS_append = " -Wno-error=pedantic" -python do_populate_cve_db () { - import subprocess - import time - - if d.getVar("BB_NO_NETWORK", True) == "1": - bb.error("BB_NO_NETWORK is set; Can't update cve-check-tool database, " - "CVEs won't be checked") +do_populate_cve_db() { + if [ "${BB_NO_NETWORK}" = "1" ] ; then + bberror "BB_NO_NETWORK is set; Can't update cve-check-tool database, CVEs won't be checked" return + fi - bb.utils.export_proxies(d) # In case we don't inherit cve-check class, use default values defined in the class. - cve_dir = d.getVar("CVE_CHECK_DB_DIR", True) or d.expand("${DL_DIR}/CVE_CHECK") - cve_file = d.getVar("CVE_CHECK_TMP_FILE", True) or d.expand("${TMPDIR}/cve_check") - cve_cmd = "cve-check-update" - cmd = [cve_cmd, "-d", cve_dir] - bb.debug(1, "Updating cve-check-tool database located in %s" % cve_dir) - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - bb.debug(2, "Command '%s' returned:\n%s" % ("\n".join(cmd), output)) - time_utc = time.gmtime(time.time()) - time_format = "%Y-%m-%d %H:%M:%S" - with open(cve_file, "w") as f: - f.write("CVE database was updated on %s UTC\n\n" - % time.strftime(time_format, time_utc)) - - except subprocess.CalledProcessError as e: - bb.warn("Error in executing cve-check-update: %s (output %s)" % (e, e.output)) - if bb.data.inherits_class('cve-check', d): - bb.warn("Failed to update cve-check-tool database, CVEs won't be checked") + cve_dir="${CVE_CHECK_DB_DIR}" + cve_file="${CVE_CHECK_TMP_FILE}" + + [ -z "${cve_dir}" ] && cve_dir="${DL_DIR}/CVE_CHECK" + [ -z "${cve_file}" ] && cve_file="${TMPDIR}/cve_check" + + bbdebug 2 "Updating cve-check-tool database located in $cve_dir" + if cve-check-update -d "$cve_dir" ; then + printf "CVE database was updated on %s UTC\n\n" "$(LANG=C date --utc +'%F %T')" > "$cve_file" + else + bbwarn "Error in executing cve-check-update" + if [ "${@'1' if bb.data.inherits_class('cve-check', d) else '0'}" -ne 0 ] ; then + bbwarn "Failed to update cve-check-tool database, CVEs won't be checked" + fi + fi } addtask populate_cve_db after do_populate_sysroot -- 2.9.3 ^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 2/2] cve-check-tool: report progress when downloading CVE database 2016-09-26 15:42 [PATCH 0/2] cve-check-tool progress reporting André Draszik 2016-09-26 15:42 ` [PATCH 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh André Draszik @ 2016-09-26 15:42 ` André Draszik 2016-09-28 10:54 ` Burton, Ross 2016-09-28 12:05 ` [PATCH v2 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh André Draszik 2 siblings, 1 reply; 8+ messages in thread From: André Draszik @ 2016-09-26 15:42 UTC (permalink / raw) To: openembedded-core We add a patch to report the progress, and at the same time inform bitbake that progress can be extracted via the simple 'percent' progress handler. Signed-off-by: André Draszik <git@andred.net> --- .../cve-check-tool/cve-check-tool_5.6.4.bb | 2 + ...ogress-in-percent-when-downloading-CVE-db.patch | 132 +++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch diff --git a/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb b/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb index 5bb22d1..1baadea 100644 --- a/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb +++ b/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb @@ -8,6 +8,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=e8c1458438ead3c34974bc0be3a03ed6" SRC_URI = "https://github.com/ikeydoherty/${BPN}/releases/download/v${PV}/${BP}.tar.xz \ file://check-for-malloc_trim-before-using-it.patch \ + file://0001-print-progress-in-percent-when-downloading-CVE-db.patch \ " SRC_URI[md5sum] = "c5f4247140fc9be3bf41491d31a34155" @@ -48,5 +49,6 @@ do_populate_cve_db() { addtask populate_cve_db after do_populate_sysroot do_populate_cve_db[nostamp] = "1" +do_populate_cve_db[progress] = "percent" BBCLASSEXTEND = "native nativesdk" diff --git a/meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch b/meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch new file mode 100644 index 0000000..ac85962 --- /dev/null +++ b/meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch @@ -0,0 +1,132 @@ +From 9dc7e787295801aaecfa744b222d58df6b1f48c2 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Andr=C3=A9=20Draszik?= <git@andred.net> +Date: Mon, 26 Sep 2016 12:12:41 +0100 +Subject: [PATCH] print progress in percent when downloading CVE db +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Upstream-Status: Pending +Signed-off-by: André Draszik <git@andred.net> +--- + src/library/fetch.c | 27 ++++++++++++++++++++++++++- + src/library/fetch.h | 2 +- + src/update.c | 15 +++++++++++---- + 3 files changed, 38 insertions(+), 6 deletions(-) + +diff --git a/src/library/fetch.c b/src/library/fetch.c +index 06d4b30..cce8e8f 100644 +--- a/src/library/fetch.c ++++ b/src/library/fetch.c +@@ -37,13 +37,36 @@ static size_t write_func(void *ptr, size_t size, size_t nmemb, struct fetch_t *f + return fwrite(ptr, size, nmemb, f->f); + } + +-FetchStatus fetch_uri(const char *uri, const char *target, bool verbose) ++struct percent_t { ++ unsigned long start; ++ unsigned long end; ++}; ++ ++static int progress_callback_new(void *ptr, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) ++{ ++ (void) ultotal; ++ (void) ulnow; ++ ++ struct percent_t *percent = (struct percent_t *) ptr; ++ ++ if (dltotal && percent) { ++ long diff = percent->end - percent->start; ++ if (diff) { ++ fprintf(stderr,"completed: %lu%%\r", percent->start + (diff * dlnow / dltotal)); ++ } ++ } ++ ++ return 0; ++} ++ ++FetchStatus fetch_uri(const char *uri, const char *target, bool verbose, unsigned long start_percent, unsigned long end_percent) + { + FetchStatus ret = FETCH_STATUS_FAIL; + CURLcode res; + struct stat st; + CURL *curl = NULL; + struct fetch_t *f = NULL; ++ struct percent_t percent = { .start = start_percent, .end = end_percent }; + + curl = curl_easy_init(); + if (!curl) { +@@ -67,6 +90,8 @@ FetchStatus fetch_uri(const char *uri, const char *target, bool verbose) + } + if (verbose) { + (void)curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); ++ (void)curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &percent); ++ (void)curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback_new); + } + res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (curl_write_callback)write_func); + if (res != CURLE_OK) { +diff --git a/src/library/fetch.h b/src/library/fetch.h +index 70c3779..70aecb7 100644 +--- a/src/library/fetch.h ++++ b/src/library/fetch.h +@@ -28,7 +28,7 @@ typedef enum { + * @param verbose Whether to be verbose + * @return A FetchStatus, indicating the operation taken + */ +-FetchStatus fetch_uri(const char *uri, const char *target, bool verbose); ++FetchStatus fetch_uri(const char *uri, const char *target, bool verbose, unsigned long this_percent, unsigned long next_percent); + + /** + * Attempt to extract the given gzipped file +diff --git a/src/update.c b/src/update.c +index 30fbe96..cd04512 100644 +--- a/src/update.c ++++ b/src/update.c +@@ -266,7 +266,7 @@ static inline void update_end(int fd, const char *update_fname, bool ok) + } + + static int do_fetch_update(int year, const char *db_dir, CveDB *cve_db, +- bool db_exist, bool verbose) ++ bool db_exist, bool verbose, unsigned long this_percent, unsigned long next_percent) + { + const char nvd_uri[] = URI_PREFIX; + autofree(cve_string) *uri_meta = NULL; +@@ -330,14 +330,14 @@ refetch: + } + + /* Fetch NVD META file */ +- st = fetch_uri(uri_meta->str, nvdcve_meta->str, verbose); ++ st = fetch_uri(uri_meta->str, nvdcve_meta->str, verbose, this_percent, this_percent); + if (st == FETCH_STATUS_FAIL) { + fprintf(stderr, "Failed to fetch %s\n", uri_meta->str); + return -1; + } + + /* Fetch NVD XML file */ +- st = fetch_uri(uri_data_gz->str, nvdcve_data_gz->str, verbose); ++ st = fetch_uri(uri_data_gz->str, nvdcve_data_gz->str, verbose, this_percent, next_percent); + switch (st) { + case FETCH_STATUS_FAIL: + fprintf(stderr, "Failed to fetch %s\n", uri_data_gz->str); +@@ -459,10 +459,17 @@ bool update_db(bool quiet, const char *db_file) + for (int i = YEAR_START; i <= year+1; i++) { + int y = i > year ? -1 : i; + int rc; ++ unsigned long start_percent = (i+0 - YEAR_START) * 100 / (year+2 - YEAR_START); ++ unsigned long end_percent = (i+1 - YEAR_START) * 100 / (year+2 - YEAR_START); + +- rc = do_fetch_update(y, db_dir, cve_db, db_exist, !quiet); ++ if (!quiet) ++ fprintf(stderr, "completed: %lu%%\r", start_percent); ++ rc = do_fetch_update(y, db_dir, cve_db, db_exist, !quiet, ++ start_percent, end_percent); + switch (rc) { + case 0: ++ if (!quiet) ++ fprintf(stderr,"completed: %lu%%\r", end_percent); + continue; + case ENOMEM: + goto oom; +-- +2.9.3 + -- 2.9.3 ^ permalink raw reply related [flat|nested] 8+ messages in thread
* Re: [PATCH 2/2] cve-check-tool: report progress when downloading CVE database 2016-09-26 15:42 ` [PATCH 2/2] cve-check-tool: report progress when downloading CVE database André Draszik @ 2016-09-28 10:54 ` Burton, Ross 2016-09-28 11:56 ` André Draszik 0 siblings, 1 reply; 8+ messages in thread From: Burton, Ross @ 2016-09-28 10:54 UTC (permalink / raw) To: André Draszik; +Cc: OE-core [-- Attachment #1: Type: text/plain, Size: 1179 bytes --] On 26 September 2016 at 16:42, André Draszik <git@andred.net> wrote: > ++static int progress_callback_new(void *ptr, curl_off_t dltotal, > curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) > ++{ > ++ (void) ultotal; > ++ (void) ulnow; > ++ > ++ struct percent_t *percent = (struct percent_t *) ptr; > ++ > ++ if (dltotal && percent) { > ++ long diff = percent->end - percent->start; > ++ if (diff) { > ++ fprintf(stderr,"completed: %lu%%\r", > percent->start + (diff * dlnow / dltotal)); > ++ } > ++ } > ++ > ++ return 0; > ++} > This fails on the autobuilder cluster: | ../../cve-check-tool-5.6.4/src/library/fetch.c: In function 'progress_callback_new': | ../../cve-check-tool-5.6.4/src/library/fetch.c:55:54: error: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'curl_off_t {aka long long int}' [-Werror=format=] | fprintf(stderr,"completed: %lu%%\r", percent->start + (diff * dlnow / dltotal)); | ^ Ross [-- Attachment #2: Type: text/html, Size: 1894 bytes --] ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 2/2] cve-check-tool: report progress when downloading CVE database 2016-09-28 10:54 ` Burton, Ross @ 2016-09-28 11:56 ` André Draszik 2016-09-28 11:57 ` Burton, Ross 0 siblings, 1 reply; 8+ messages in thread From: André Draszik @ 2016-09-28 11:56 UTC (permalink / raw) To: Burton, Ross; +Cc: OE-core Ups, sorry for that... :-( Should I send a patch against master-next, or against master? a. On Mi, 2016-09-28 at 11:54 +0100, Burton, Ross wrote: > On 26 September 2016 at 16:42, André Draszik <git@andred.net> wrote: > > > > > ++static int progress_callback_new(void *ptr, curl_off_t dltotal, > > curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) > > ++{ > > ++ (void) ultotal; > > ++ (void) ulnow; > > ++ > > ++ struct percent_t *percent = (struct percent_t *) ptr; > > ++ > > ++ if (dltotal && percent) { > > ++ long diff = percent->end - percent->start; > > ++ if (diff) { > > ++ fprintf(stderr,"completed: %lu%%\r", > > percent->start + (diff * dlnow / dltotal)); > > ++ } > > ++ } > > ++ > > ++ return 0; > > ++} > > > > This fails on the autobuilder cluster: > > > > > ../../cve-check-tool-5.6.4/src/library/fetch.c: In function > 'progress_callback_new': > > > > ../../cve-check-tool-5.6.4/src/library/fetch.c:55:54: error: format > > '%lu' > expects argument of type 'long unsigned int', but argument 3 has type > 'curl_off_t {aka long long int}' [-Werror=format=] > > > > fprintf(stderr,"completed: %lu%%\r", > percent->start + (diff * dlnow / dltotal)); > > > > ^ > > Ross ^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 2/2] cve-check-tool: report progress when downloading CVE database 2016-09-28 11:56 ` André Draszik @ 2016-09-28 11:57 ` Burton, Ross 0 siblings, 0 replies; 8+ messages in thread From: Burton, Ross @ 2016-09-28 11:57 UTC (permalink / raw) To: André Draszik; +Cc: OE-core [-- Attachment #1: Type: text/plain, Size: 262 bytes --] On 28 September 2016 at 12:56, André Draszik <git@andred.net> wrote: > Ups, sorry for that... :-( > > Should I send a patch against master-next, or against master? > Happens to the best of us :) Edit your patch and resend it against master. Ross [-- Attachment #2: Type: text/html, Size: 678 bytes --] ^ permalink raw reply [flat|nested] 8+ messages in thread
* [PATCH v2 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh 2016-09-26 15:42 [PATCH 0/2] cve-check-tool progress reporting André Draszik 2016-09-26 15:42 ` [PATCH 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh André Draszik 2016-09-26 15:42 ` [PATCH 2/2] cve-check-tool: report progress when downloading CVE database André Draszik @ 2016-09-28 12:05 ` André Draszik 2016-09-28 12:05 ` [PATCH v2 2/2] cve-check-tool: report progress when downloading CVE database André Draszik 2 siblings, 1 reply; 8+ messages in thread From: André Draszik @ 2016-09-28 12:05 UTC (permalink / raw) To: openembedded-core This will allow us to easily incorporate progress support via bb.process.run() Signed-off-by: André Draszik <git@andred.net> --- .../cve-check-tool/cve-check-tool_5.6.4.bb | 45 +++++++++------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb b/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb index 1165559..5bb22d1 100644 --- a/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb +++ b/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb @@ -22,35 +22,28 @@ inherit pkgconfig autotools EXTRA_OECONF = "--disable-coverage" CFLAGS_append = " -Wno-error=pedantic" -python do_populate_cve_db () { - import subprocess - import time - - if d.getVar("BB_NO_NETWORK", True) == "1": - bb.error("BB_NO_NETWORK is set; Can't update cve-check-tool database, " - "CVEs won't be checked") +do_populate_cve_db() { + if [ "${BB_NO_NETWORK}" = "1" ] ; then + bberror "BB_NO_NETWORK is set; Can't update cve-check-tool database, CVEs won't be checked" return + fi - bb.utils.export_proxies(d) # In case we don't inherit cve-check class, use default values defined in the class. - cve_dir = d.getVar("CVE_CHECK_DB_DIR", True) or d.expand("${DL_DIR}/CVE_CHECK") - cve_file = d.getVar("CVE_CHECK_TMP_FILE", True) or d.expand("${TMPDIR}/cve_check") - cve_cmd = "cve-check-update" - cmd = [cve_cmd, "-d", cve_dir] - bb.debug(1, "Updating cve-check-tool database located in %s" % cve_dir) - try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - bb.debug(2, "Command '%s' returned:\n%s" % ("\n".join(cmd), output)) - time_utc = time.gmtime(time.time()) - time_format = "%Y-%m-%d %H:%M:%S" - with open(cve_file, "w") as f: - f.write("CVE database was updated on %s UTC\n\n" - % time.strftime(time_format, time_utc)) - - except subprocess.CalledProcessError as e: - bb.warn("Error in executing cve-check-update: %s (output %s)" % (e, e.output)) - if bb.data.inherits_class('cve-check', d): - bb.warn("Failed to update cve-check-tool database, CVEs won't be checked") + cve_dir="${CVE_CHECK_DB_DIR}" + cve_file="${CVE_CHECK_TMP_FILE}" + + [ -z "${cve_dir}" ] && cve_dir="${DL_DIR}/CVE_CHECK" + [ -z "${cve_file}" ] && cve_file="${TMPDIR}/cve_check" + + bbdebug 2 "Updating cve-check-tool database located in $cve_dir" + if cve-check-update -d "$cve_dir" ; then + printf "CVE database was updated on %s UTC\n\n" "$(LANG=C date --utc +'%F %T')" > "$cve_file" + else + bbwarn "Error in executing cve-check-update" + if [ "${@'1' if bb.data.inherits_class('cve-check', d) else '0'}" -ne 0 ] ; then + bbwarn "Failed to update cve-check-tool database, CVEs won't be checked" + fi + fi } addtask populate_cve_db after do_populate_sysroot -- 2.9.3 ^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH v2 2/2] cve-check-tool: report progress when downloading CVE database 2016-09-28 12:05 ` [PATCH v2 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh André Draszik @ 2016-09-28 12:05 ` André Draszik 0 siblings, 0 replies; 8+ messages in thread From: André Draszik @ 2016-09-28 12:05 UTC (permalink / raw) To: openembedded-core We add a patch to report the progress, and at the same time inform bitbake that progress can be extracted via the simple 'percent' progress handler. Signed-off-by: André Draszik <git@andred.net> --- .../cve-check-tool/cve-check-tool_5.6.4.bb | 2 + ...ogress-in-percent-when-downloading-CVE-db.patch | 135 +++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch diff --git a/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb b/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb index 5bb22d1..1baadea 100644 --- a/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb +++ b/meta/recipes-devtools/cve-check-tool/cve-check-tool_5.6.4.bb @@ -8,6 +8,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=e8c1458438ead3c34974bc0be3a03ed6" SRC_URI = "https://github.com/ikeydoherty/${BPN}/releases/download/v${PV}/${BP}.tar.xz \ file://check-for-malloc_trim-before-using-it.patch \ + file://0001-print-progress-in-percent-when-downloading-CVE-db.patch \ " SRC_URI[md5sum] = "c5f4247140fc9be3bf41491d31a34155" @@ -48,5 +49,6 @@ do_populate_cve_db() { addtask populate_cve_db after do_populate_sysroot do_populate_cve_db[nostamp] = "1" +do_populate_cve_db[progress] = "percent" BBCLASSEXTEND = "native nativesdk" diff --git a/meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch b/meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch new file mode 100644 index 0000000..0510e3a --- /dev/null +++ b/meta/recipes-devtools/cve-check-tool/files/0001-print-progress-in-percent-when-downloading-CVE-db.patch @@ -0,0 +1,135 @@ +From e9ed26cde63f8ca7607a010a518329339f8c02d3 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Andr=C3=A9=20Draszik?= <git@andred.net> +Date: Mon, 26 Sep 2016 12:12:41 +0100 +Subject: [PATCH] print progress in percent when downloading CVE db +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Upstream-Status: Pending +Signed-off-by: André Draszik <git@andred.net> +--- + src/library/fetch.c | 28 +++++++++++++++++++++++++++- + src/library/fetch.h | 3 ++- + src/update.c | 16 ++++++++++++---- + 3 files changed, 41 insertions(+), 6 deletions(-) + +diff --git a/src/library/fetch.c b/src/library/fetch.c +index 06d4b30..0fe6d76 100644 +--- a/src/library/fetch.c ++++ b/src/library/fetch.c +@@ -37,13 +37,37 @@ static size_t write_func(void *ptr, size_t size, size_t nmemb, struct fetch_t *f + return fwrite(ptr, size, nmemb, f->f); + } + +-FetchStatus fetch_uri(const char *uri, const char *target, bool verbose) ++struct percent_t { ++ unsigned int start; ++ unsigned int end; ++}; ++ ++static int progress_callback_new(void *ptr, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) ++{ ++ (void) ultotal; ++ (void) ulnow; ++ ++ struct percent_t *percent = (struct percent_t *) ptr; ++ ++ if (dltotal && percent && percent->end >= percent->start) { ++ unsigned int diff = percent->end - percent->start; ++ if (diff) { ++ fprintf(stderr,"completed: "CURL_FORMAT_OFF_T"%%\r", percent->start + (diff * dlnow / dltotal)); ++ } ++ } ++ ++ return 0; ++} ++ ++FetchStatus fetch_uri(const char *uri, const char *target, bool verbose, ++ unsigned int start_percent, unsigned int end_percent) + { + FetchStatus ret = FETCH_STATUS_FAIL; + CURLcode res; + struct stat st; + CURL *curl = NULL; + struct fetch_t *f = NULL; ++ struct percent_t percent = { .start = start_percent, .end = end_percent }; + + curl = curl_easy_init(); + if (!curl) { +@@ -67,6 +91,8 @@ FetchStatus fetch_uri(const char *uri, const char *target, bool verbose) + } + if (verbose) { + (void)curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); ++ (void)curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &percent); ++ (void)curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_callback_new); + } + res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, (curl_write_callback)write_func); + if (res != CURLE_OK) { +diff --git a/src/library/fetch.h b/src/library/fetch.h +index 70c3779..4cce5d1 100644 +--- a/src/library/fetch.h ++++ b/src/library/fetch.h +@@ -28,7 +28,8 @@ typedef enum { + * @param verbose Whether to be verbose + * @return A FetchStatus, indicating the operation taken + */ +-FetchStatus fetch_uri(const char *uri, const char *target, bool verbose); ++FetchStatus fetch_uri(const char *uri, const char *target, bool verbose, ++ unsigned int this_percent, unsigned int next_percent); + + /** + * Attempt to extract the given gzipped file +diff --git a/src/update.c b/src/update.c +index 30fbe96..eaeeefd 100644 +--- a/src/update.c ++++ b/src/update.c +@@ -266,7 +266,8 @@ static inline void update_end(int fd, const char *update_fname, bool ok) + } + + static int do_fetch_update(int year, const char *db_dir, CveDB *cve_db, +- bool db_exist, bool verbose) ++ bool db_exist, bool verbose, ++ unsigned int this_percent, unsigned int next_percent) + { + const char nvd_uri[] = URI_PREFIX; + autofree(cve_string) *uri_meta = NULL; +@@ -330,14 +331,14 @@ refetch: + } + + /* Fetch NVD META file */ +- st = fetch_uri(uri_meta->str, nvdcve_meta->str, verbose); ++ st = fetch_uri(uri_meta->str, nvdcve_meta->str, verbose, this_percent, this_percent); + if (st == FETCH_STATUS_FAIL) { + fprintf(stderr, "Failed to fetch %s\n", uri_meta->str); + return -1; + } + + /* Fetch NVD XML file */ +- st = fetch_uri(uri_data_gz->str, nvdcve_data_gz->str, verbose); ++ st = fetch_uri(uri_data_gz->str, nvdcve_data_gz->str, verbose, this_percent, next_percent); + switch (st) { + case FETCH_STATUS_FAIL: + fprintf(stderr, "Failed to fetch %s\n", uri_data_gz->str); +@@ -459,10 +460,17 @@ bool update_db(bool quiet, const char *db_file) + for (int i = YEAR_START; i <= year+1; i++) { + int y = i > year ? -1 : i; + int rc; ++ unsigned int start_percent = ((i+0 - YEAR_START) * 100) / (year+2 - YEAR_START); ++ unsigned int end_percent = ((i+1 - YEAR_START) * 100) / (year+2 - YEAR_START); + +- rc = do_fetch_update(y, db_dir, cve_db, db_exist, !quiet); ++ if (!quiet) ++ fprintf(stderr, "completed: %u%%\r", start_percent); ++ rc = do_fetch_update(y, db_dir, cve_db, db_exist, !quiet, ++ start_percent, end_percent); + switch (rc) { + case 0: ++ if (!quiet) ++ fprintf(stderr,"completed: %u%%\r", end_percent); + continue; + case ENOMEM: + goto oom; +-- +2.9.3 + -- 2.9.3 ^ permalink raw reply related [flat|nested] 8+ messages in thread
end of thread, other threads:[~2016-09-28 12:05 UTC | newest] Thread overview: 8+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2016-09-26 15:42 [PATCH 0/2] cve-check-tool progress reporting André Draszik 2016-09-26 15:42 ` [PATCH 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh André Draszik 2016-09-26 15:42 ` [PATCH 2/2] cve-check-tool: report progress when downloading CVE database André Draszik 2016-09-28 10:54 ` Burton, Ross 2016-09-28 11:56 ` André Draszik 2016-09-28 11:57 ` Burton, Ross 2016-09-28 12:05 ` [PATCH v2 1/2] cve-check-tool: convert do_populate_cve_db() from python to sh André Draszik 2016-09-28 12:05 ` [PATCH v2 2/2] cve-check-tool: report progress when downloading CVE database André Draszik
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.