Reduce file transfers to speed up runtime (#1232)

rsyncing > 100K files from GCS on a single-core node was proving very slow, so use a tarball instead. I also picked up an error in the documentation, which had been transposed into the code, so the desired file wasn't even being retained :-(
google · Apr 19, 2023 · e276155 · e276155
1 parent 920997a
commit e276155
Show file tree

Hide file tree

Showing 7 changed files with 17 additions and 11 deletions.
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/debian-copyright-mirror.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/debian-copyright-mirror.yaml
@@ -11,6 +11,6 @@ spec:
           - name: debian-copyright-mirror
             env:
             - name: GCS_PATH
-              value: gs://osv-test-cve-osv-conversion/debian_copyright
+              value: gs://osv-test-cve-osv-conversion/debian_copyright/debian_copyright.tar
             - name: BE_VERBOSE
               value:
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/gen-cperepos-map.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/gen-cperepos-map.yaml
@@ -11,7 +11,7 @@ spec:
           - name: gen-cperepos-map
             env:
             - name: DEBIAN_COPYRIGHT_GCS_PATH
-              value: gs://osv-test-cve-osv-conversion/debian_copyright
+              value: gs://osv-test-cve-osv-conversion/debian_copyright/debian_copyright.tar
             - name: CPEREPO_GCS_PATH
               value: gs://osv-test-cve-osv-conversion/cpe_repos
             - name: BE_VERBOSE

diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/debian-copyright-mirror.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/debian-copyright-mirror.yaml
@@ -11,4 +11,4 @@ spec:
           - name: debian-copyright-mirror
             env:
             - name: GCS_PATH
-              value: gs://cve-osv-conversion/debian_copyright
+              value: gs://cve-osv-conversion/debian_copyright/debian_copyright.tar
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/gen-cperepos-map.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/gen-cperepos-map.yaml
@@ -11,6 +11,6 @@ spec:
           - name: gen-cperepos-map
             env:
             - name: DEBIAN_COPYRIGHT_GCS_PATH
-              value: gs://cve-osv-conversion/debian_copyright
+              value: gs://cve-osv-conversion/debian_copyright/debian_copyright.tar
             - name: CPEREPO_GCS_PATH
               value: gs://cve-osv-conversion/cpe_repos
diff --git a/vulnfeeds/cmd/cperepos/README.md b/vulnfeeds/cmd/cperepos/README.md
@@ -9,7 +9,7 @@ metadata mirror with:
 wget \
   --directory debian_copyright \
     --mirror \
-    -A debian_copyright \
+    -A unstable_copyright \
     -A index.html \
     https://metadata.ftp-master.debian.org/changelogs/main
 ```

diff --git a/vulnfeeds/cmd/cperepos/gen_cperepos_map b/vulnfeeds/cmd/cperepos/gen_cperepos_map
@@ -33,7 +33,8 @@ curl ${BE_VERBOSE="--silent"} \
 
 MAYBE_USE_DEBIAN_COPYRIGHT_METADATA=""
 if [[ -n "${DEBIAN_COPYRIGHT_GCS_PATH}" ]]; then
-  gsutil ${BE_VERBOSE="-q"} -m rsync -r "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}"
+  gsutil ${BE_VERBOSE="-q"} cp "${DEBIAN_COPYRIGHT_GCS_PATH}" "${WORK_DIR}"
+  tar -C "${WORK_DIR}" -xf "${WORK_DIR}/$(basename ${DEBIAN_COPYRIGHT_GCS_PATH})"
   MAYBE_USE_DEBIAN_COPYRIGHT_METADATA="--debian_metadata_path ${WORK_DIR}/metadata.ftp-master.debian.org"
 fi
 
@@ -42,4 +43,4 @@ fi
   ${MAYBE_USE_DEBIAN_COPYRIGHT_METADATA} \
   --output_dir "${WORK_DIR}"
 
-gsutil ${BE_VERBOSE="-q"} rsync -c -d -r "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}"
+gsutil ${BE_VERBOSE="-q"} cp "${WORK_DIR}/cpe_product_to_repo.json" "${CPEREPO_GCS_PATH}"
diff --git a/vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror b/vulnfeeds/cmd/debian-copyright-mirror/debian-copyright-mirror
@@ -21,22 +21,27 @@
 #
 # Inputs:
 # * A local work directory
-# * GCS bucket name + path
+# * GCS bucket name + path to tarball
 #
 
 # Setting BE_VERBOSE to an empty string or null value suppresses silencing of
 # commands
 
 mkdir -p "${WORK_DIR}" || true
 
-gsutil ${BE_VERBOSE="-q"} -m rsync -d -r "${GCS_PATH}" "${WORK_DIR}"
+if gsutil --quiet stat "${GCS_PATH}"; then
+  gsutil ${BE_VERBOSE="--quiet"} cp "${GCS_PATH}" "${WORK_DIR}"
+  tar -C "${WORK_DIR}" -xf "${WORK_DIR}/$(basename ${GCS_PATH})"
+fi
 
 wget \
   ${BE_VERBOSE="--quiet"} \
   --directory "${WORK_DIR}" \
   --mirror \
-  --accept debian_copyright \
+  --accept unstable_copyright \
   --accept index.html \
   https://metadata.ftp-master.debian.org/changelogs/main
 
-gsutil ${BE_VERBOSE="-q"} -m rsync -d -r "${WORK_DIR}" "${GCS_PATH}"
+tar -C "${WORK_DIR}" -cf "${WORK_DIR}/$(basename ${GCS_PATH})" .
+
+gsutil ${BE_VERBOSE="--quiet"} cp "${WORK_DIR}/$(basename ${GCS_PATH})" "${GCS_PATH}"