Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: store last modified timestamp in db #6271

Merged
merged 1 commit into from Dec 12, 2023
Merged

fix: store last modified timestamp in db #6271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
20 changes: 20 additions & 0 deletions core/src/main/java/org/owasp/dependencycheck/data/nvdcve/DatabaseProperties.java
View file
Open in desktop
Expand Up @@ -28,6 +28,7 @@
import javax.annotation.concurrent.ThreadSafe;

import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.utils.DateUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -72,6 +73,14 @@ public class DatabaseProperties {
* The key for the last check time for the Known Exploited Vulnerabilities.
*/
public static final String KEV_LAST_CHECKED = "kev.checked";
/**
* The key for the last check time for the Retire JS repository.
*/
public static final String RETIRE_LAST_CHECKED = "retirejs.checked";
/**
* The key for the last check time for the hosted suppression file.
*/
public static final String HOSTED_SUPPRESSION_LAST_CHECKED = "hosted.suppression.checked";
/**
* The key for the version the Known Exploited Vulnerabilities.
*/
Expand Down Expand Up @@ -214,5 +223,16 @@ public static ZonedDateTime getTimestamp(Properties properties, String key) {
}
return null;
}

/**
* Returns the database property value in seconds.
*
* @param key the key to the property
* @return the property value in seconds
*/
public long getPropertyInSeconds(String key) {
final String value = getProperty(key, "0");
return DateUtil.getEpochValueInSeconds(value);
}

}
38 changes: 29 additions & 9 deletions core/src/main/java/org/owasp/dependencycheck/data/update/HostedSuppressionsDataSource.java
View file
Open in desktop
Expand Up @@ -33,8 +33,9 @@
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;

public class HostedSuppressionsDataSource implements CachedWebDataSource {
public class HostedSuppressionsDataSource implements CachedWebDataSource {

/**
* Static logger.
Expand All @@ -45,7 +46,10 @@ public class HostedSuppressionsDataSource implements CachedWebDataSource {
* The configured settings.
*/
private Settings settings;

/**
* The properties obtained from the database.
*/
private DatabaseProperties dbProperties = null;
/**
* The default URL to the Hosted Suppressions file.
*/
Expand All @@ -55,12 +59,17 @@ public class HostedSuppressionsDataSource implements CachedWebDataSource {
* Downloads the current Hosted suppressions file.
*
* @param engine a reference to the ODC Engine
* @return returns false as no updates are made to the database, just web resources cached locally
* @return returns false as no updates are made to the database, just web
* resources cached locally
* @throws UpdateException thrown if the update encountered fatal errors
*/
@Override
public boolean update(Engine engine) throws UpdateException {
this.settings = engine.getSettings();
if (engine.getMode() != Engine.Mode.EVIDENCE_COLLECTION) {
//note this conditional is only to support test cases.
this.dbProperties = engine.getDatabase().getDatabaseProperties();
}
final String configuredUrl = settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_URL, DEFAULT_SUPPRESSIONS_URL);
final boolean autoupdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE, true);
final boolean forceupdate = settings.getBoolean(Settings.KEYS.HOSTED_SUPPRESSIONS_FORCEUPDATE, false);
Expand All @@ -76,11 +85,14 @@ public boolean update(Engine engine) throws UpdateException {
if (proceed) {
LOGGER.debug("Begin Hosted Suppressions file update");
fetchHostedSuppressions(settings, url, repoFile);
if (dbProperties != null) {
dbProperties.save(DatabaseProperties.HOSTED_SUPPRESSION_LAST_CHECKED, Long.toString(System.currentTimeMillis() / 1000));
}
}
} catch (UpdateException ex) {
// only emit a warning, DependencyCheck will continue without taking the latest hosted suppressions into account.
LOGGER.warn("Failed to update hosted suppressions file, results may contain false positives already resolved by the "
+ "DependencyCheck project", ex);
+ "DependencyCheck project", ex);
} catch (MalformedURLException ex) {
throw new UpdateException(String.format("Invalid URL for Hosted Suppressions file (%s)", configuredUrl), ex);
} catch (IOException ex) {
Expand All @@ -93,16 +105,23 @@ public boolean update(Engine engine) throws UpdateException {
* Determines if the we should update the Hosted Suppressions file.
*
* @param repo the Hosted Suppressions file.
* @return <code>true</code> if an update to the Hosted Suppressions file should
* be performed; otherwise <code>false</code>
* @return <code>true</code> if an update to the Hosted Suppressions file
* should be performed; otherwise <code>false</code>
* @throws NumberFormatException thrown if an invalid value is contained in
* the database properties
*/
protected boolean shouldUpdate(File repo) throws NumberFormatException {
boolean proceed = true;
if (repo != null && repo.isFile()) {
final int validForHours = settings.getInt(Settings.KEYS.HOSTED_SUPPRESSIONS_VALID_FOR_HOURS, 2);
final long lastUpdatedOn = repo.lastModified();
long lastUpdatedOn = 0;
if (dbProperties != null) {
lastUpdatedOn = dbProperties.getPropertyInSeconds(DatabaseProperties.HOSTED_SUPPRESSION_LAST_CHECKED);
}
if (lastUpdatedOn <= 0) {
//fall back on conversion from file last modified to storing in the db.
lastUpdatedOn = repo.lastModified();
}
final long now = System.currentTimeMillis();
LOGGER.debug("Last updated: {}", lastUpdatedOn);
LOGGER.debug("Now: {}", now);
Expand All @@ -120,7 +139,8 @@ protected boolean shouldUpdate(File repo) throws NumberFormatException {
*
* @param settings a reference to the dependency-check settings
* @param repoUrl the URL to the hosted suppressions file to use
* @param repoFile the local file where the hosted suppressions file is to be placed
* @param repoFile the local file where the hosted suppressions file is to
* be placed
* @throws UpdateException thrown if there is an exception during
* initialization
*/
Expand All @@ -144,7 +164,7 @@ public boolean purge(Engine engine) {
boolean result = true;
try {
final URL repoUrl = new URL(settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_URL,
DEFAULT_SUPPRESSIONS_URL));
DEFAULT_SUPPRESSIONS_URL));
final String filename = new File(repoUrl.getPath()).getName();
final File repo = new File(settings.getDataDirectory(), filename);
if (repo.exists()) {
Expand Down
13 changes: 1 addition & 12 deletions core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java
View file
Open in desktop
Expand Up @@ -120,7 +120,7 @@ private boolean shouldUpdate() throws UpdateException {
if (cveDB.dataExists() && 0 < validForHours) {
// ms Valid = valid (hours) x 60 min/hour x 60 sec/min x 1000 ms/sec
final long validForSeconds = validForHours * 60L * 60L;
final long lastChecked = getPropertyInSeconds(DatabaseProperties.KEV_LAST_CHECKED);
final long lastChecked = dbProperties.getPropertyInSeconds(DatabaseProperties.KEV_LAST_CHECKED);
final long now = System.currentTimeMillis() / 1000;
proceed = (now - lastChecked) > validForSeconds;
if (!proceed) {
Expand All @@ -130,15 +130,4 @@ private boolean shouldUpdate() throws UpdateException {
return proceed;
}

/**
* Returns the database property value in seconds.
*
* @param key the key to the property
* @return the property value in seconds
*/
private long getPropertyInSeconds(String key) {
final String value = dbProperties.getProperty(key, "0");
return DateUtil.getEpochValueInSeconds(value);
}

}
13 changes: 12 additions & 1 deletion core/src/main/java/org/owasp/dependencycheck/data/update/RetireJSDataSource.java
View file
Open in desktop
Expand Up @@ -24,6 +24,7 @@
import javax.annotation.concurrent.ThreadSafe;

import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties;
import org.owasp.dependencycheck.data.update.exception.UpdateException;
import org.owasp.dependencycheck.exception.WriteLockException;
import org.owasp.dependencycheck.utils.Downloader;
Expand Down Expand Up @@ -54,6 +55,10 @@ public class RetireJSDataSource implements CachedWebDataSource {
* The configured settings.
*/
private Settings settings;
/**
* The properties obtained from the database.
*/
private DatabaseProperties dbProperties = null;
/**
* The default URL to the RetireJS JavaScript repository.
*/
Expand All @@ -75,6 +80,7 @@ public RetireJSDataSource() {
@Override
public boolean update(Engine engine) throws UpdateException {
this.settings = engine.getSettings();
this.dbProperties = engine.getDatabase().getDatabaseProperties();
final String configuredUrl = settings.getString(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, DEFAULT_JS_URL);
final boolean autoupdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE, true);
final boolean forceupdate = settings.getBoolean(Settings.KEYS.ANALYZER_RETIREJS_FORCEUPDATE, false);
Expand All @@ -87,6 +93,7 @@ public boolean update(Engine engine) throws UpdateException {
if (proceed) {
LOGGER.debug("Begin RetireJS Update");
initializeRetireJsRepo(settings, url, repoFile);
dbProperties.save(DatabaseProperties.RETIRE_LAST_CHECKED, Long.toString(System.currentTimeMillis() / 1000));
}
} catch (MalformedURLException ex) {
throw new UpdateException(String.format("Invalid URL for RetireJS repository (%s)", configuredUrl), ex);
Expand All @@ -109,7 +116,11 @@ protected boolean shouldUpdate(File repo) throws NumberFormatException {
boolean proceed = true;
if (repo != null && repo.isFile()) {
final int validForHours = settings.getInt(Settings.KEYS.ANALYZER_RETIREJS_REPO_VALID_FOR_HOURS, 0);
final long lastUpdatedOn = repo.lastModified();
long lastUpdatedOn = dbProperties.getPropertyInSeconds(DatabaseProperties.RETIRE_LAST_CHECKED);
if (lastUpdatedOn <= 0) {
//fall back on conversion from file last modified to storing in the db.
lastUpdatedOn = repo.lastModified();
}
final long now = System.currentTimeMillis();
LOGGER.debug("Last updated: {}", lastUpdatedOn);
LOGGER.debug("Now: {}", now);
Expand Down