fix: use temporary files to reduce memory usage

core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java

@@ -17,10 +17,13 @@
  */
 package org.owasp.dependencycheck.data.update;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClient;
 import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.StringReader;
 import java.net.MalformedURLException;
@@ -41,6 +44,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.zip.GZIPOutputStream;
 import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
@@ -321,14 +325,21 @@ private boolean processApi() throws UpdateException {
             int ctr = 0;
             try (NvdCveClient api = builder.build()) {
                 while (api.hasNext()) {
-                    final Collection<DefCveItem> items = api.next();
+                    Collection<DefCveItem> items = api.next();
                     max = api.getTotalAvailable();
                     if (ctr == 0) {
                         LOGGER.info(String.format("NVD API has %,d records in this update", max));
                     }
                     if (items != null && !items.isEmpty()) {
-                        final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, items));
+                        final ObjectMapper objectMapper = new ObjectMapper();
+                        objectMapper.registerModule(new JavaTimeModule());
+                        final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz");
+                        try (FileOutputStream fos = new FileOutputStream(outputFile);
+                                GZIPOutputStream out = new GZIPOutputStream(fos);) {
+                        objectMapper.writeValue(out, items);
+                        final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
                         submitted.add(f);
+                        }
                         ctr += 1;
                         if ((ctr % 5) == 0) {
                             final double percent = (double) (ctr * RESULTS_PER_PAGE) / max * 100;

core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java

@@ -17,17 +17,14 @@
  */
 package org.owasp.dependencycheck.data.update.nvd.api;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20;
+import java.io.File;
 import java.net.URL;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import javax.annotation.concurrent.ThreadSafe;
 import org.apache.commons.lang3.StringUtils;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
-import org.owasp.dependencycheck.data.update.exception.UpdateException;
 import org.owasp.dependencycheck.utils.Downloader;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;
@@ -72,7 +69,6 @@ public class DownloadTask implements Callable<Future<NvdApiProcessor>> {
      * @param settings a reference to the global settings object; this is
      * necessary so that when the thread is started the dependencies have a
      * correct reference to the global settings.
-     * @throws UpdateException thrown if temporary files could not be created
      */
     public DownloadTask(String url, ExecutorService processor, CveDB cveDB, Settings settings) {
         this.url = url;
@@ -89,15 +85,12 @@ public Future<NvdApiProcessor> call() throws Exception {
             LOGGER.info("Download Started for NVD Cache - {}", url);
             final long startDownload = System.currentTimeMillis();
             final Downloader d = new Downloader(settings);
-            final String content = d.fetchGzContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
-            final ObjectMapper objectMapper = new ObjectMapper();
-            objectMapper.registerModule(new JavaTimeModule());
-            final CveApiJson20 data = objectMapper.readValue(content, CveApiJson20.class);
-
+            final File outputFile = settings.getTempFile("nvd-datafeed-", "json.gz");
+            d.fetchFile(u, outputFile, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
             if (this.processorService == null) {
                 return null;
             }
-            final NvdApiProcessor task = new NvdApiProcessor(cveDB, data.getVulnerabilities(), startDownload);
+            final NvdApiProcessor task = new NvdApiProcessor(cveDB, outputFile, startDownload);
             final Future<NvdApiProcessor> val = this.processorService.submit(task);
             return val;
         } catch (Throwable ex) {

core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java

@@ -17,11 +17,20 @@
  */
 package org.owasp.dependencycheck.data.update.nvd.api;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20;
 import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
 import java.util.Collection;
 import java.util.concurrent.Callable;
+import java.util.zip.GZIPInputStream;
 import org.owasp.dependencycheck.data.nvd.ecosystem.CveEcosystemMapper;
 import org.owasp.dependencycheck.data.nvdcve.CveDB;
+import org.owasp.dependencycheck.data.update.exception.UpdateException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,9 +50,9 @@ public class NvdApiProcessor implements Callable<NvdApiProcessor> {
      */
     private final CveDB cveDB;
     /**
-     * The collection of NVD API data to add to the database.
+     * The file containing the data to inject.
      */
-    private Collection<DefCveItem> data;
+    private File jsonFile;
     /**
      * Reference to the CVE Ecosystem Mapper object.
      */
@@ -60,37 +69,62 @@ public class NvdApiProcessor implements Callable<NvdApiProcessor> {
     /**
      * Create a new processor to put the NVD data into the database.
      *
-     * @param cveDB a reference to the database
-     * @param data the data to add to the database
+     * @param cveDB a reference to the database.
+     * @param jsonFile the JSON data file to inject.
      * @param startTime the start time of the update process.
      */
-    public NvdApiProcessor(final CveDB cveDB, Collection<DefCveItem> data, long startTime) {
+    public NvdApiProcessor(final CveDB cveDB, File jsonFile, long startTime) {
         this.cveDB = cveDB;
-        this.data = data;
+        this.jsonFile = jsonFile;
         this.startTime = startTime;
     }
 
     /**
      * Create a new processor to put the NVD data into the database.
      *
      * @param cveDB a reference to the database
-     * @param data the data to add to the database
+     * @param jsonFile the JSON data file to inject.
      */
-    public NvdApiProcessor(final CveDB cveDB, Collection<DefCveItem> data) {
-        this(cveDB, data, System.currentTimeMillis());
+    public NvdApiProcessor(final CveDB cveDB, File jsonFile) {
+        this(cveDB, jsonFile, System.currentTimeMillis());
     }
 
     @Override
     public NvdApiProcessor call() throws Exception {
-        for (DefCveItem entry : data) {
-            try {
-            cveDB.updateVulnerability(entry, mapper.getEcosystem(entry));
-            } catch (Exception ex) {
-                LOGGER.error("Failed to process " + entry.getCve().getId(), ex);
+        final ObjectMapper objectMapper = new ObjectMapper();
+        objectMapper.registerModule(new JavaTimeModule());
+         Collection<DefCveItem> data = null;
+
+         if (jsonFile.getName().endsWith(".jsonarray.gz")) {
+            try (FileInputStream fileInputStream = new FileInputStream(jsonFile);
+                        GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) {
+                data = objectMapper.readValue(gzipInputStream, new TypeReference<Collection<DefCveItem>>(){});
+            } catch (IOException exception) {
+                throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception);
+            }
+        } else if (jsonFile.getName().endsWith(".gz")) {
+            try (FileInputStream fileInputStream = new FileInputStream(jsonFile);
+                        GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) {
+                CveApiJson20 cveData = objectMapper.readValue(gzipInputStream, CveApiJson20.class);
+                if (cveData != null) {
+                    data = cveData.getVulnerabilities();
+                }
+            } catch (IOException exception) {
+                throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception);
+            }
+        } else {
+            data = objectMapper.readValue(jsonFile, new TypeReference<Collection<DefCveItem>>(){});
+        }
+        if (data != null ) {
+            for (DefCveItem entry : data) {
+                try {
+                    cveDB.updateVulnerability(entry, mapper.getEcosystem(entry));
+                } catch (Exception ex) {
+                    LOGGER.error("Failed to process " + entry.getCve().getId(), ex);
+                }
             }
         }
         endTime = System.currentTimeMillis();
-        data = null;
         return this;
     }
 

utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java

@@ -178,7 +178,7 @@ public String fetchContent(URL url, boolean useProxy) throws DownloadFailedExcep
     public String fetchContent(URL url, boolean useProxy, String userKey, String passwordKey)
             throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
         InputStream in = null;
-        try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey);
+        try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey); 
                 ByteArrayOutputStream out = new ByteArrayOutputStream()) {
             in = conn.fetch(url);
             IOUtils.copy(in, out);
@@ -196,42 +196,4 @@ public String fetchContent(URL url, boolean useProxy, String userKey, String pas
             }
         }
     }
-
-    /**
-     * Retrieves a gzip file from a given URL and returns the uncompressed contents.
-     *
-     * @param url the URL of the file to download
-     * @param useProxy whether to use the configured proxy when downloading
-     * files
-     * @return the content of the file
-     * @param userKey the settings key for the username to be used
-     * @param passwordKey the settings key for the password to be used
-     * @throws DownloadFailedException is thrown if there is an error
-     * downloading the file
-     * @throws TooManyRequestsException thrown when a 429 is received
-     * @throws ResourceNotFoundException thrown when a 404 is received
-     */
-    public String fetchGzContent(URL url, boolean useProxy, String userKey, String passwordKey)
-            throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException {
-        InputStream in = null;
-        try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey)) {
-            in = conn.fetch(url);
-            try (GZIPInputStream gzipIn = new GZIPInputStream(in);
-                    ByteArrayOutputStream out = new ByteArrayOutputStream()) {
-                IOUtils.copy(gzipIn, out);
-                return out.toString(UTF8);
-            }
-        } catch (IOException ex) {
-            final String msg = format("Download failed, unable to retrieve '%s'; %s", url, ex.getMessage());
-            throw new DownloadFailedException(msg, ex);
-        } finally {
-            if (in != null) {
-                try {
-                    in.close();
-                } catch (IOException ex) {
-                    LOGGER.trace("Ignorable error", ex);
-                }
-            }
-        }
-    }
 }