diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java index 9b9518c63fa..a8d4ba0729b 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java @@ -171,13 +171,13 @@ public synchronized Map getMetaData() { if (!"version".equals(key)) { if (DatabaseProperties.NVD_API_LAST_CHECKED.equals(key)) { map.put("NVD API Last Checked", entry.getValue().toString()); - + } else if (DatabaseProperties.NVD_API_LAST_MODIFIED.equals(key)) { map.put("NVD API Last Modified", entry.getValue().toString()); - + } else if (DatabaseProperties.NVD_CACHE_LAST_CHECKED.equals(key)) { map.put("NVD Cache Last Checked", entry.getValue().toString()); - + } else if (DatabaseProperties.NVD_CACHE_LAST_MODIFIED.equals(key)) { map.put("NVD Cache Last Modified", entry.getValue().toString()); } @@ -207,6 +207,18 @@ public void save(String key, ZonedDateTime timestamp) throws UpdateException { save(key, dtf.format(timestamp)); } + /** + * Stores a timestamp in the properties file. + * + * @param properties the properties to store the timestamp + * @param key the property key + * @param timestamp the zoned date time + */ + public static void setTimestamp(Properties properties, String key, ZonedDateTime timestamp) throws UpdateException { + final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX"); + properties.put(key, dtf.format(timestamp)); + } + /** * Retrieves a zoned date time. * @@ -223,7 +235,25 @@ public static ZonedDateTime getTimestamp(Properties properties, String key) { } return null; } - + + /** + * Retrieves a zoned date time. + * + * @param properties the properties file containing the date time + * @param key the property key + * @return the zoned date time + */ + public static ZonedDateTime getIsoTimestamp(Properties properties, String key) { + //final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX"); + final DateTimeFormatter dtf = DateTimeFormatter.ISO_DATE_TIME; + final String val = properties.getProperty(key); + if (val != null) { + final String value = properties.getProperty(key); + return ZonedDateTime.parse(value, dtf); + } + return null; + } + /** * Returns the database property value in seconds. * diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java index bf3ad2ffbbb..f8b779c0e2d 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java @@ -26,7 +26,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.StringReader; -import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.text.MessageFormat; import java.time.Duration; @@ -138,7 +139,7 @@ private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException { final UrlData data = extractUrlData(nvdDataFeedUrl); String url = data.getUrl(); String pattern = data.getPattern(); - final Properties cacheProperties = getRemoteCacheProperties(url); + final Properties cacheProperties = getRemoteCacheProperties(url, pattern); if (pattern == null) { final String prefix = cacheProperties.getProperty("prefix", "nvdcve-"); pattern = prefix + "{0}.json.gz"; @@ -341,11 +342,10 @@ private boolean processApi() throws UpdateException { final ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new JavaTimeModule()); final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz"); - try (FileOutputStream fos = new FileOutputStream(outputFile); - GZIPOutputStream out = new GZIPOutputStream(fos);) { - objectMapper.writeValue(out, items); - final Future f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile)); - submitted.add(f); + try (FileOutputStream fos = new FileOutputStream(outputFile); GZIPOutputStream out = new GZIPOutputStream(fos);) { + objectMapper.writeValue(out, items); + final Future f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile)); + submitted.add(f); } ctr += 1; if ((ctr % 5) == 0) { @@ -562,21 +562,51 @@ protected final Map getUpdatesNeeded(String url, String filePatt * @throws UpdateException thrown if the properties file could not be * downloaded */ - protected final Properties getRemoteCacheProperties(String url) throws UpdateException { + protected final Properties getRemoteCacheProperties(String url, String pattern) throws UpdateException { + final Downloader d = new Downloader(settings); + final Properties properties = new Properties(); try { - final URL u = new URL(url + "cache.properties"); - final Downloader d = new Downloader(settings); + final URL u = new URI(url + "cache.properties").toURL(); final String content = d.fetchContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); - final Properties properties = new Properties(); properties.load(new StringReader(content)); - return properties; - } catch (MalformedURLException ex) { + + } catch (URISyntaxException ex) { throw new UpdateException("Invalid NVD Cache URL", ex); - } catch (DownloadFailedException | TooManyRequestsException | ResourceNotFoundException ex) { + } catch (DownloadFailedException | ResourceNotFoundException ex) { + String metaPattern; + if (pattern == null) { + metaPattern = "nvdcve-{0}.meta"; + } else { + metaPattern = pattern.replace(".json.gz", ".meta"); + } + try { + URL metaUrl = new URI(url + MessageFormat.format(metaPattern, "modified")).toURL(); + String content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + Properties props = new Properties(); + props.load(new StringReader(content)); + ZonedDateTime lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate"); + DatabaseProperties.setTimestamp(properties,"lastModifiedDate.modified", lmd); + DatabaseProperties.setTimestamp(properties,"lastModifiedDate", lmd); + final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002); + final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); + final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear(); + for (int y = startYear; y <= endYear; y++) { + metaUrl = new URI(url + MessageFormat.format(metaPattern, String.valueOf(y))).toURL(); + content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + props.clear(); + props.load(new StringReader(content)); + lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate"); + DatabaseProperties.setTimestamp(properties, "lastModifiedDate." + String.valueOf(y), lmd); + } + } catch (URISyntaxException | TooManyRequestsException | ResourceNotFoundException | IOException ex1) { + throw new UpdateException("Unable to download the data feed META files", ex); + } + } catch ( TooManyRequestsException ex) { throw new UpdateException("Unable to download the NVD API cache.properties", ex); } catch (IOException ex) { throw new UpdateException("Invalid NVD Cache Properties file contents", ex); } + return properties; } protected static class UrlData {