Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: read meta files if cache.properties does not exist #6282

Merged
merged 1 commit into from Dec 12, 2023
Merged

fix: read meta files if cache.properties does not exist #6282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
38 changes: 34 additions & 4 deletions core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java
View file
Open in desktop
Expand Up @@ -171,13 +171,13 @@ public synchronized Map<String, String> getMetaData() {
if (!"version".equals(key)) {
if (DatabaseProperties.NVD_API_LAST_CHECKED.equals(key)) {
map.put("NVD API Last Checked", entry.getValue().toString());

} else if (DatabaseProperties.NVD_API_LAST_MODIFIED.equals(key)) {
map.put("NVD API Last Modified", entry.getValue().toString());

} else if (DatabaseProperties.NVD_CACHE_LAST_CHECKED.equals(key)) {
map.put("NVD Cache Last Checked", entry.getValue().toString());

} else if (DatabaseProperties.NVD_CACHE_LAST_MODIFIED.equals(key)) {
map.put("NVD Cache Last Modified", entry.getValue().toString());
}
Expand Down Expand Up @@ -207,6 +207,18 @@ public void save(String key, ZonedDateTime timestamp) throws UpdateException {
save(key, dtf.format(timestamp));
}

/**
* Stores a timestamp in the properties file.
*
* @param properties the properties to store the timestamp
* @param key the property key
* @param timestamp the zoned date time
*/
public static void setTimestamp(Properties properties, String key, ZonedDateTime timestamp) throws UpdateException {
final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX");
properties.put(key, dtf.format(timestamp));
}

/**
* Retrieves a zoned date time.
*
Expand All @@ -223,7 +235,25 @@ public static ZonedDateTime getTimestamp(Properties properties, String key) {
}
return null;
}


/**
* Retrieves a zoned date time.
*
* @param properties the properties file containing the date time
* @param key the property key
* @return the zoned date time
*/
public static ZonedDateTime getIsoTimestamp(Properties properties, String key) {
//final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ssX");
final DateTimeFormatter dtf = DateTimeFormatter.ISO_DATE_TIME;
final String val = properties.getProperty(key);
if (val != null) {
final String value = properties.getProperty(key);
return ZonedDateTime.parse(value, dtf);
}
return null;
}

/**
* Returns the database property value in seconds.
*
Expand Down
58 changes: 44 additions & 14 deletions core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java
View file
Open in desktop
Expand Up @@ -26,7 +26,8 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.text.MessageFormat;
import java.time.Duration;
Expand Down Expand Up @@ -138,7 +139,7 @@ private boolean processDatafeed(String nvdDataFeedUrl) throws UpdateException {
final UrlData data = extractUrlData(nvdDataFeedUrl);
String url = data.getUrl();
String pattern = data.getPattern();
final Properties cacheProperties = getRemoteCacheProperties(url);
final Properties cacheProperties = getRemoteCacheProperties(url, pattern);
if (pattern == null) {
final String prefix = cacheProperties.getProperty("prefix", "nvdcve-");
pattern = prefix + "{0}.json.gz";
Expand Down Expand Up @@ -341,11 +342,10 @@ private boolean processApi() throws UpdateException {
final ObjectMapper objectMapper = new ObjectMapper();
objectMapper.registerModule(new JavaTimeModule());
final File outputFile = settings.getTempFile("nvd-data-", ".jsonarray.gz");
try (FileOutputStream fos = new FileOutputStream(outputFile);
GZIPOutputStream out = new GZIPOutputStream(fos);) {
objectMapper.writeValue(out, items);
final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
submitted.add(f);
try (FileOutputStream fos = new FileOutputStream(outputFile); GZIPOutputStream out = new GZIPOutputStream(fos);) {
objectMapper.writeValue(out, items);
final Future<NvdApiProcessor> f = processingExecutorService.submit(new NvdApiProcessor(cveDb, outputFile));
submitted.add(f);
}
ctr += 1;
if ((ctr % 5) == 0) {
Expand Down Expand Up @@ -562,21 +562,51 @@ protected final Map<String, String> getUpdatesNeeded(String url, String filePatt
* @throws UpdateException thrown if the properties file could not be
* downloaded
*/
protected final Properties getRemoteCacheProperties(String url) throws UpdateException {
protected final Properties getRemoteCacheProperties(String url, String pattern) throws UpdateException {
final Downloader d = new Downloader(settings);
final Properties properties = new Properties();
try {
final URL u = new URL(url + "cache.properties");
final Downloader d = new Downloader(settings);
final URL u = new URI(url + "cache.properties").toURL();
final String content = d.fetchContent(u, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
final Properties properties = new Properties();
properties.load(new StringReader(content));
return properties;
} catch (MalformedURLException ex) {

} catch (URISyntaxException ex) {
throw new UpdateException("Invalid NVD Cache URL", ex);
} catch (DownloadFailedException | TooManyRequestsException | ResourceNotFoundException ex) {
} catch (DownloadFailedException | ResourceNotFoundException ex) {
String metaPattern;
if (pattern == null) {
metaPattern = "nvdcve-{0}.meta";
} else {
metaPattern = pattern.replace(".json.gz", ".meta");
}
try {
URL metaUrl = new URI(url + MessageFormat.format(metaPattern, "modified")).toURL();
String content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
Properties props = new Properties();
props.load(new StringReader(content));
ZonedDateTime lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
DatabaseProperties.setTimestamp(properties,"lastModifiedDate.modified", lmd);
DatabaseProperties.setTimestamp(properties,"lastModifiedDate", lmd);
final int startYear = settings.getInt(Settings.KEYS.NVD_API_DATAFEED_START_YEAR, 2002);
final ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
final int endYear = now.withZoneSameInstant(ZoneId.of("UTC+14:00")).getYear();
for (int y = startYear; y <= endYear; y++) {
metaUrl = new URI(url + MessageFormat.format(metaPattern, String.valueOf(y))).toURL();
content = d.fetchContent(metaUrl, true, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD);
props.clear();
props.load(new StringReader(content));
lmd = DatabaseProperties.getIsoTimestamp(props, "lastModifiedDate");
DatabaseProperties.setTimestamp(properties, "lastModifiedDate." + String.valueOf(y), lmd);
}
} catch (URISyntaxException | TooManyRequestsException | ResourceNotFoundException | IOException ex1) {
throw new UpdateException("Unable to download the data feed META files", ex);
}
} catch ( TooManyRequestsException ex) {
throw new UpdateException("Unable to download the NVD API cache.properties", ex);
} catch (IOException ex) {
throw new UpdateException("Invalid NVD Cache Properties file contents", ex);
}
return properties;
}

protected static class UrlData {
Expand Down