Skip to content

Commit

Permalink
fix: read meta files if cache.properties does not exist (#6282)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeremylong
jeremylong committed Dec 12, 2023
1 parent 4b3287b commit 6769966
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 18 deletions.
38 changes: 34 additions & 4 deletions core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java
View file
Expand Up @@ -171,13 +171,13 @@ public synchronized Map<String, String> getMetaData() {
if (!"version".equals(key)) {
if (DatabaseProperties.NVD_API_LAST_CHECKED.equals(key)) {
map.put("NVD API Last Checked", entry.getValue().toString());

} else if (DatabaseProperties.NVD_API_LAST_MODIFIED.equals(key)) {
map.put("NVD API Last Modified", entry.getValue().toString());

} else if (DatabaseProperties.NVD_CACHE_LAST_CHECKED.equals(key)) {
map.put("NVD Cache Last Checked", entry.getValue().toString());

} else if (DatabaseProperties.NVD_CACHE_LAST_MODIFIED.equals(key)) {
map.put("NVD Cache Last Modified", entry.getValue().toString());
}
Expand Down Expand Up @@ -207,6 +207,18 @@ public void save(String key, ZonedDateTime timestamp) throws UpdateException {
save(key, dtf.format(timestamp));
}

/**
* Stores a timestamp in the properties file.
*
* @param properties the properties to store the timestamp
* @param key the property key
* @param timestamp the zoned date time
*/
public static void setTimestamp(Properties properties, String key, ZonedDateTime timestamp) throws UpdateException {
final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX");
properties.put(key, dtf.format(timestamp));
}

/**
* Retrieves a zoned date time.
*
Expand All @@ -223,7 +235,25 @@ public static ZonedDateTime getTimestamp(Properties properties, String key) {
}
return null;
}


/**
* Retrieves a zoned date time.
*
* @param properties the properties file containing the date time
* @param key the property key
* @return the zoned date time
*/
public static ZonedDateTime getIsoTimestamp(Properties properties, String key) {
//final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX");
final DateTimeFormatter dtf = DateTimeFormatter.ISO_DATE_TIME;
final String val = properties.getProperty(key);
if (val != null) {
final String value = properties.getProperty(key);
return ZonedDateTime.parse(value, dtf);
}
return null;
}

/**
* Returns the database property value in seconds.
*
Expand Down
58 changes: 44 additions & 14 deletions core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java
View file
Expand Up @@ -26,7 +26,8 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.text.MessageFormat;
import java.time.Duration;
Expand Down Expand Up @@ -138,7 +139,7 @@ private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException {
final UrlData data = extractUrlData(nvdDataFeedUrl);
String url = data.getUrl();
String pattern = data.getPattern();
final Properties cacheProperties = getRemoteCacheProperties(url);
final Properties cacheProperties = getRemoteCacheProperties(url, pattern);
if (pattern == null) {
final String prefix = cacheProperties.getProperty("prefix", "nvdcve-");
pattern = prefix + "{0}.json.gz";
Expand Down Expand Up @@ -341,11 +342,10 @@ private boolean processApi() throws UpdateException {
final ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz");
try (FileOutputStream fos = new FileOutputStream(outputFile);
GZIPOutputStream out = new GZIPOutputStream(fos);) {
objectMapper.writeValue(out, items);
final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
submitted.add(f);
try (FileOutputStream fos = new FileOutputStream(outputFile); GZIPOutputStream out = new GZIPOutputStream(fos);) {
objectMapper.writeValue(out, items);
final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
submitted.add(f);
}
ctr += 1;
if ((ctr % 5) == 0) {
Expand Down Expand Up @@ -562,21 +562,51 @@ protected final Map<String, String> getUpdatesNeeded(String url, String filePatt
* @throws UpdateException thrown if the properties file could not be
* downloaded
*/
protected final Properties getRemoteCacheProperties(String url) throws UpdateException {
protected final Properties getRemoteCacheProperties(String url, String pattern) throws UpdateException {
final Downloader d = new Downloader(settings);
final Properties properties = new Properties();
try {
final URL u = new URL(url + "cache.properties");
final Downloader d = new Downloader(settings);
final URL u = new URI(url + "cache.properties").toURL();
final String content = d.fetchContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
final Properties properties = new Properties();
properties.load(new StringReader(content));
return properties;
} catch (MalformedURLException ex) {

} catch (URISyntaxException ex) {
throw new UpdateException("Invalid NVD Cache URL", ex);
} catch (DownloadFailedException | TooManyRequestsException | ResourceNotFoundException ex) {
} catch (DownloadFailedException | ResourceNotFoundException ex) {
String metaPattern;
if (pattern == null) {
metaPattern = "nvdcve-{0}.meta";
} else {
metaPattern = pattern.replace(".json.gz", ".meta");
}
try {
URL metaUrl = new URI(url + MessageFormat.format(metaPattern, "modified")).toURL();
String content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
Properties props = new Properties();
props.load(new StringReader(content));
ZonedDateTime lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
DatabaseProperties.setTimestamp(properties,"lastModifiedDate.modified", lmd);
DatabaseProperties.setTimestamp(properties,"lastModifiedDate", lmd);
final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
for (int y = startYear; y <= endYear; y++) {
metaUrl = new URI(url + MessageFormat.format(metaPattern, String.valueOf(y))).toURL();
content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
props.clear();
props.load(new StringReader(content));
lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
DatabaseProperties.setTimestamp(properties, "lastModifiedDate." + String.valueOf(y), lmd);
}
} catch (URISyntaxException | TooManyRequestsException | ResourceNotFoundException | IOException ex1) {
throw new UpdateException("Unable to download the data feed META files", ex);
}
} catch ( TooManyRequestsException ex) {
throw new UpdateException("Unable to download the NVD API cache.properties", ex);
} catch (IOException ex) {
throw new UpdateException("Invalid NVD Cache Properties file contents", ex);
}
return properties;
}

protected static class UrlData {
Expand Down

0 comments on commit 6769966

Please sign in to comment.