From 0b1261c42e57a24245908b433db45540bd5b6146 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 9 Dec 2023 09:12:26 -0500 Subject: [PATCH 1/2] fix: use temporary files to reduce memory usage resolves #6196 --- .../data/update/NvdApiDataSource.java | 15 ++++- .../data/update/nvd/api/DownloadTask.java | 11 ++-- .../data/update/nvd/api/NvdApiProcessor.java | 65 ++++++++++++++----- .../dependencycheck/utils/Downloader.java | 40 +----------- 4 files changed, 68 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java index bae465e38d4..4b6d776906f 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java @@ -17,10 +17,13 @@ */ package org.owasp.dependencycheck.data.update; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClient; import io.github.jeremylong.openvulnerability.client.nvd.NvdCveClientBuilder; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.StringReader; import java.net.MalformedURLException; @@ -41,6 +44,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.zip.GZIPOutputStream; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.data.nvdcve.CveDB; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; @@ -321,14 +325,21 @@ private boolean processApi() throws UpdateException { int ctr = 0; try (NvdCveClient api = builder.build()) { while (api.hasNext()) { - final Collection items = api.next(); + Collection items = api.next(); max = api.getTotalAvailable(); if (ctr == 0) { LOGGER.info(String.format("NVD API has %,d records in this update", max)); } if (items != null && !items.isEmpty()) { - final Future f = processingExecutorService.submit(new NvdApiProcessor(cveDb, items)); + final ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz"); + try (FileOutputStream fos = new FileOutputStream(outputFile); + GZIPOutputStream out = new GZIPOutputStream(fos);) { + objectMapper.writeValue(out, items); + final Future f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile)); submitted.add(f); + } ctr += 1; if ((ctr % 5) == 0) { final double percent = (double) (ctr * RESULTS_PER_PAGE) / max * 100; diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java index 1bbbb89790f..4799eb0ddfe 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20; +import java.io.File; import java.net.URL; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; @@ -72,7 +73,6 @@ public class DownloadTask implements Callable> { * @param settings a reference to the global settings object; this is * necessary so that when the thread is started the dependencies have a * correct reference to the global settings. - * @throws UpdateException thrown if temporary files could not be created */ public DownloadTask(String url, ExecutorService processor, CveDB cveDB, Settings settings) { this.url = url; @@ -89,15 +89,12 @@ public Future call() throws Exception { LOGGER.info("Download Started for NVD Cache - {}", url); final long startDownload = System.currentTimeMillis(); final Downloader d = new Downloader(settings); - final String content = d.fetchGzContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); - final ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new JavaTimeModule()); - final CveApiJson20 data = objectMapper.readValue(content, CveApiJson20.class); - + final File outputFile = settings.getTempFile("nvd-datafeed-", "json.gz"); + d.fetchFile(u, outputFile, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); if (this.processorService == null) { return null; } - final NvdApiProcessor task = new NvdApiProcessor(cveDB, data.getVulnerabilities(), startDownload); + final NvdApiProcessor task = new NvdApiProcessor(cveDB, outputFile, startDownload); final Future val = this.processorService.submit(task); return val; } catch (Throwable ex) { diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java index c4304fe4b64..374ae44d083 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java @@ -17,11 +17,21 @@ */ package org.owasp.dependencycheck.data.update.nvd.api; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20; import io.github.jeremylong.openvulnerability.client.nvd.DefCveItem; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.concurrent.Callable; +import java.util.zip.GZIPInputStream; import org.owasp.dependencycheck.data.nvd.ecosystem.CveEcosystemMapper; import org.owasp.dependencycheck.data.nvdcve.CveDB; +import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,9 +51,9 @@ public class NvdApiProcessor implements Callable { */ private final CveDB cveDB; /** - * The collection of NVD API data to add to the database. + * The file containing the data to inject. */ - private Collection data; + private File jsonFile; /** * Reference to the CVE Ecosystem Mapper object. */ @@ -60,13 +70,13 @@ public class NvdApiProcessor implements Callable { /** * Create a new processor to put the NVD data into the database. * - * @param cveDB a reference to the database - * @param data the data to add to the database + * @param cveDB a reference to the database. + * @param jsonFile the JSON data file to inject. * @param startTime the start time of the update process. */ - public NvdApiProcessor(final CveDB cveDB, Collection data, long startTime) { + public NvdApiProcessor(final CveDB cveDB, File jsonFile, long startTime) { this.cveDB = cveDB; - this.data = data; + this.jsonFile = jsonFile; this.startTime = startTime; } @@ -74,23 +84,48 @@ public NvdApiProcessor(final CveDB cveDB, Collection data, long star * Create a new processor to put the NVD data into the database. * * @param cveDB a reference to the database - * @param data the data to add to the database + * @param jsonFile the JSON data file to inject. */ - public NvdApiProcessor(final CveDB cveDB, Collection data) { - this(cveDB, data, System.currentTimeMillis()); + public NvdApiProcessor(final CveDB cveDB, File jsonFile) { + this(cveDB, jsonFile, System.currentTimeMillis()); } @Override public NvdApiProcessor call() throws Exception { - for (DefCveItem entry : data) { - try { - cveDB.updateVulnerability(entry, mapper.getEcosystem(entry)); - } catch (Exception ex) { - LOGGER.error("Failed to process " + entry.getCve().getId(), ex); + final ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new JavaTimeModule()); + Collection data = null; + + if (jsonFile.getName().endsWith(".jsonarray.gz")) { + try (FileInputStream fileInputStream = new FileInputStream(jsonFile); + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) { + data = objectMapper.readValue(gzipInputStream, new TypeReference>(){}); + } catch (IOException exception) { + throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception); + } + } else if (jsonFile.getName().endsWith(".gz")) { + try (FileInputStream fileInputStream = new FileInputStream(jsonFile); + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);) { + CveApiJson20 cveData = objectMapper.readValue(gzipInputStream, CveApiJson20.class); + if (cveData != null) { + data = cveData.getVulnerabilities(); + } + } catch (IOException exception) { + throw new UpdateException("Unable to read downloaded json data: " + jsonFile, exception); + } + } else { + data = objectMapper.readValue(jsonFile, new TypeReference>(){}); + } + if (data != null ) { + for (DefCveItem entry : data) { + try { + cveDB.updateVulnerability(entry, mapper.getEcosystem(entry)); + } catch (Exception ex) { + LOGGER.error("Failed to process " + entry.getCve().getId(), ex); + } } } endTime = System.currentTimeMillis(); - data = null; return this; } diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index 0dcc231acea..92a1e35105b 100755 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -178,7 +178,7 @@ public String fetchContent(URL url, boolean useProxy) throws DownloadFailedExcep public String fetchContent(URL url, boolean useProxy, String userKey, String passwordKey) throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { InputStream in = null; - try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey); + try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey); ByteArrayOutputStream out = new ByteArrayOutputStream()) { in = conn.fetch(url); IOUtils.copy(in, out); @@ -196,42 +196,4 @@ public String fetchContent(URL url, boolean useProxy, String userKey, String pas } } } - - /** - * Retrieves a gzip file from a given URL and returns the uncompressed contents. - * - * @param url the URL of the file to download - * @param useProxy whether to use the configured proxy when downloading - * files - * @return the content of the file - * @param userKey the settings key for the username to be used - * @param passwordKey the settings key for the password to be used - * @throws DownloadFailedException is thrown if there is an error - * downloading the file - * @throws TooManyRequestsException thrown when a 429 is received - * @throws ResourceNotFoundException thrown when a 404 is received - */ - public String fetchGzContent(URL url, boolean useProxy, String userKey, String passwordKey) - throws DownloadFailedException, TooManyRequestsException, ResourceNotFoundException { - InputStream in = null; - try (HttpResourceConnection conn = new HttpResourceConnection(settings, useProxy, userKey, passwordKey)) { - in = conn.fetch(url); - try (GZIPInputStream gzipIn = new GZIPInputStream(in); - ByteArrayOutputStream out = new ByteArrayOutputStream()) { - IOUtils.copy(gzipIn, out); - return out.toString(UTF8); - } - } catch (IOException ex) { - final String msg = format("Download failed, unable to retrieve '%s'; %s", url, ex.getMessage()); - throw new DownloadFailedException(msg, ex); - } finally { - if (in != null) { - try { - in.close(); - } catch (IOException ex) { - LOGGER.trace("Ignorable error", ex); - } - } - } - } } From 191c36e8cff2b41aeeac4eea595b139619819108 Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Sat, 9 Dec 2023 09:13:32 -0500 Subject: [PATCH 2/2] fix: remove unused imports --- .../dependencycheck/data/update/nvd/api/DownloadTask.java | 4 ---- .../dependencycheck/data/update/nvd/api/NvdApiProcessor.java | 1 - 2 files changed, 5 deletions(-) diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java index 4799eb0ddfe..03b26634ecf 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java @@ -17,9 +17,6 @@ */ package org.owasp.dependencycheck.data.update.nvd.api; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import io.github.jeremylong.openvulnerability.client.nvd.CveApiJson20; import java.io.File; import java.net.URL; import java.util.concurrent.Callable; @@ -28,7 +25,6 @@ import javax.annotation.concurrent.ThreadSafe; import org.apache.commons.lang3.StringUtils; import org.owasp.dependencycheck.data.nvdcve.CveDB; -import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.owasp.dependencycheck.utils.Downloader; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java index 374ae44d083..1aee82bb4d9 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/NvdApiProcessor.java @@ -26,7 +26,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.util.Collection; -import java.util.List; import java.util.concurrent.Callable; import java.util.zip.GZIPInputStream; import org.owasp.dependencycheck.data.nvd.ecosystem.CveEcosystemMapper;