From 6f972cc287b1f3e3f8b2369eeed6bb1608232401 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sun, 10 Dec 2023 05:51:39 -0500 Subject: [PATCH] fix: use temporary files to reduce memory usage (#6270) --- .../data/update/NvdApiDataSource.java | 15 ++++- .../data/update/nvd/api/DownloadTask.java | 15 ++--- .../data/update/nvd/api/NvdApiProcessor.java | 64 ++++++++++++++----- .../dependencycheck/utils/Downloader.java | 40 +----------- 4 files changed, 67 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java index 0c333e4c69f..bf3ad2ffbbb 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java @@ -17,10 +17,13 @@ */ package org.owasp.dependencycheck.data.update; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClient; import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.StringReader; import java.net.MalformedURLException; @@ -41,6 +44,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.zip.GZIPOutputStream; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; @@ -328,14 +332,21 @@ private boolean processApi() throws UpdateException { int ctr = 0; try (NvdCveClient api = builder.build()) { while (api.hasNext()) { - final Collection items = api.next(); + Collection items = api.next(); max = api.getTotalAvailable(); if (ctr == 0) { LOGGER.info(String.format("NVD API has %,d records in this update", max)); } if (items != null && !items.isEmpty()) { - final Future f = processingExecutorService.submit(new NvdApiProcessor(cveDb, items)); + final ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz"); + try (FileOutputStream fos = new FileOutputStream(outputFile); + GZIPOutputStream out = new GZIPOutputStream(fos);) { + objectMapper.writeValue(out, items); + final Future f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile)); submitted.add(f); + } ctr += 1; if ((ctr % 5) == 0) { final double percent = (double) (ctr * RESULTS_PER_PAGE) / max * 100; diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java index 1bbbb89790f..03b26634ecf 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java @@ -17,9 +17,7 @@ */ package org.owasp.dependencycheck.data.update.nvd.api; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20; +import java.io.File; import java.net.URL; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; @@ -27,7 +25,6 @@ import javax.annotation.concurrent.ThreadSafe; import org.apache.commons.lang3.StringUtils; import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; @@ -72,7 +69,6 @@ public class DownloadTask implements Callable> { * @param settings a reference to the global settings object; this is * necessary so that when the thread is started the dependencies have a * correct reference to the global settings. - * @throws UpdateException thrown if temporary files could not be created */ public DownloadTask(String url, ExecutorService processor, CveDB cveDB, Settings settings) { this.url = url; @@ -89,15 +85,12 @@ public Future call() throws Exception { LOGGER.info("Download Started for NVD Cache - {}", url); final long startDownload = System.currentTimeMillis(); final Downloader d = new Downloader(settings); - final String content = d.fetchGzContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); - final ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new JavaTimeModule()); - final CveApiJson20 data = objectMapper.readValue(content, CveApiJson20.class); - + final File outputFile = settings.getTempFile("nvd-datafeed-", "json.gz"); + d.fetchFile(u, outputFile, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); if (this.processorService == null) { return null; } - final NvdApiProcessor task = new NvdApiProcessor(cveDB, data.getVulnerabilities(), startDownload); + final NvdApiProcessor task = new NvdApiProcessor(cveDB, outputFile, startDownload); final Future val = this.processorService.submit(task); return val; } catch (Throwable ex) { diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java index c4304fe4b64..1aee82bb4d9 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java @@ -17,11 +17,20 @@ */ package org.owasp.dependencycheck.data.update.nvd.api; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20; import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.util.Collection; import java.util.concurrent.Callable; +import java.util.zip.GZIPInputStream; import org.owasp.dependencycheck.data.nvd.ecosystem.CveEcosystemMapper; import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,9 +50,9 @@ public class NvdApiProcessor implements Callable { */ private final CveDB cveDB; /** - * The collection of NVD API data to add to the database. + * The file containing the data to inject. */ - private Collection data; + private File jsonFile; /** * Reference to the CVE Ecosystem Mapper object. */ @@ -60,13 +69,13 @@ public class NvdApiProcessor implements Callable { /** * Create a new processor to put the NVD data into the database. * - * @param cveDB a reference to the database - * @param data the data to add to the database + * @param cveDB a reference to the database. + * @param jsonFile the JSON data file to inject. * @param startTime the start time of the update process. */ - public NvdApiProcessor(final CveDB cveDB, Collection data, long startTime) { + public NvdApiProcessor(final CveDB cveDB, File jsonFile, long startTime) { this.cveDB = cveDB; - this.data = data; + this.jsonFile = jsonFile; this.startTime = startTime; } @@ -74,23 +83,48 @@ public NvdApiProcessor(final CveDB cveDB, Collection data, long star * Create a new processor to put the NVD data into the database. * * @param cveDB a reference to the database - * @param data the data to add to the database + * @param jsonFile the JSON data file to inject. */ - public NvdApiProcessor(final CveDB cveDB, Collection data) { - this(cveDB, data, System.currentTimeMillis()); + public NvdApiProcessor(final CveDB cveDB, File jsonFile) { + this(cveDB, jsonFile, System.currentTimeMillis()); } @Override public NvdApiProcessor call() throws Exception { - for (DefCveItem entry : data) { - try { - cveDB.updateVulnerability(entry, mapper.getEcosystem(entry)); - } catch (Exception ex) { - LOGGER.error("Failed to process " + entry.getCve().getId(), ex); + final ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + Collection data = null; + + if (jsonFile.getName().endsWith(".jsonarray.gz")) { + try (FileInputStream fileInputStream = new FileInputStream(jsonFile); + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) { + data = objectMapper.readValue(gzipInputStream, new TypeReference>(){}); + } catch (IOException exception) { + throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception); + } + } else if (jsonFile.getName().endsWith(".gz")) { + try (FileInputStream fileInputStream = new FileInputStream(jsonFile); + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) { + CveApiJson20 cveData = objectMapper.readValue(gzipInputStream, CveApiJson20.class); + if (cveData != null) { + data = cveData.getVulnerabilities(); + } + } catch (IOException exception) { + throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception); + } + } else { + data = objectMapper.readValue(jsonFile, new TypeReference>(){}); + } + if (data != null ) { + for (DefCveItem entry : data) { + try { + cveDB.updateVulnerability(entry, mapper.getEcosystem(entry)); + } catch (Exception ex) { + LOGGER.error("Failed to process " + entry.getCve().getId(), ex); + } } } endTime = System.currentTimeMillis(); - data = null; return this; } diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index 0dcc231acea..92a1e35105b 100755 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -178,7 +178,7 @@ public String fetchContent(URL url, boolean useProxy) throws DownloadFailedExcep public String fetchContent(URL url, boolean useProxy, String userKey, String passwordKey) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { InputStream in = null; - try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey); + try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey); ByteArrayOutputStream out = new ByteArrayOutputStream()) { in = conn.fetch(url); IOUtils.copy(in, out); @@ -196,42 +196,4 @@ public String fetchContent(URL url, boolean useProxy, String userKey, String pas } } } - - /** - * Retrieves a gzip file from a given URL and returns the uncompressed contents. - * - * @param url the URL of the file to download - * @param useProxy whether to use the configured proxy when downloading - * files - * @return the content of the file - * @param userKey the settings key for the username to be used - * @param passwordKey the settings key for the password to be used - * @throws DownloadFailedException is thrown if there is an error - * downloading the file - * @throws TooManyRequestsException thrown when a 429 is received - * @throws ResourceNotFoundException thrown when a 404 is received - */ - public String fetchGzContent(URL url, boolean useProxy, String userKey, String passwordKey) - throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - InputStream in = null; - try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey)) { - in = conn.fetch(url); - try (GZIPInputStream gzipIn = new GZIPInputStream(in); - ByteArrayOutputStream out = new ByteArrayOutputStream()) { - IOUtils.copy(gzipIn, out); - return out.toString(UTF8); - } - } catch (IOException ex) { - final String msg = format("Download failed, unable to retrieve '%s'; %s", url, ex.getMessage()); - throw new DownloadFailedException(msg, ex); - } finally { - if (in != null) { - try { - in.close(); - } catch (IOException ex) { - LOGGER.trace("Ignorable error", ex); - } - } - } - } }