Relaunch "Update last modified time of combine-to-osv entries" (#2165)

Same as #2158
Merge after weekly release and
#2164

vulnfeeds/cmd/combine-to-osv/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path"
 	"strings"
+	"time"
 
 	"github.com/google/osv/vulnfeeds/cves"
 	"github.com/google/osv/vulnfeeds/utility"
@@ -42,13 +43,31 @@ func main() {
 	}
 
 	allCves := loadAllCVEs(*cvePath)
-	allParts := loadParts(*partsInputPath)
-	combinedData := combineIntoOSV(allCves, allParts, *cveListPath)
+	allParts, cveModifiedMap := loadParts(*partsInputPath)
+	combinedData := combineIntoOSV(allCves, allParts, *cveListPath, cveModifiedMap)
 	writeOSVFile(combinedData, *osvOutputPath)
 }
 
+// getModifiedTime gets the modification time of a given file
+// This function assumes that the modified time on disk matches with it in GCS
+func getModifiedTime(filePath string) (time.Time, error) {
+	var emptyTime time.Time
+	fileInfo, err := os.Stat(filePath)
+	if err != nil {
+		return emptyTime, err
+	}
+	parsedTime := fileInfo.ModTime()
+
+	return parsedTime, err
+}
+
 // loadInnerParts loads second level folder for the loadParts function
-func loadInnerParts(innerPartInputPath string, output map[cves.CVEID][]vulns.PackageInfo) {
+//
+// Parameters:
+//   - innerPartInputPath: The inner part path, such as "parts/alpine"
+//   - output: A map to store all PackageInfos for each CVE ID
+//   - cvePartsModifiedTime: A map tracking the latest modification time of each CVE part files
+func loadInnerParts(innerPartInputPath string, output map[cves.CVEID][]vulns.PackageInfo, cvePartsModifiedTime map[cves.CVEID]time.Time) {
 	dirInner, err := os.ReadDir(innerPartInputPath)
 	if err != nil {
 		Logger.Fatalf("Failed to read dir %q: %s", innerPartInputPath, err)
@@ -57,7 +76,8 @@ func loadInnerParts(innerPartInputPath string, output map[cves.CVEID][]vulns.Pac
 		if !strings.HasSuffix(entryInner.Name(), ".json") {
 			continue
 		}
-		file, err := os.Open(path.Join(innerPartInputPath, entryInner.Name()))
+		filePath := path.Join(innerPartInputPath, entryInner.Name())
+		file, err := os.Open(filePath)
 		if err != nil {
 			Logger.Fatalf("Failed to open PackageInfo JSON %q: %s", path.Join(innerPartInputPath, entryInner.Name()), err)
 		}
@@ -74,6 +94,17 @@ func loadInnerParts(innerPartInputPath string, output map[cves.CVEID][]vulns.Pac
 
 		Logger.Infof(
 			"Loaded Item: %s", entryInner.Name())
+
+		// Updates the latest OSV parts modified time of each CVE
+		modifiedTime, err := getModifiedTime(filePath)
+		if err != nil {
+			Logger.Warnf("Failed to get modified time of %s: %s", filePath, err)
+			continue
+		}
+		existingDate, exists := cvePartsModifiedTime[cveId]
+		if !exists || modifiedTime.After(existingDate) {
+			cvePartsModifiedTime[cveId] = modifiedTime
+		}
 	}
 }
 
@@ -90,25 +121,27 @@ func loadInnerParts(innerPartInputPath string, output map[cves.CVEID][]vulns.Pac
 //
 // ## Returns
 // A mapping of "CVE-ID": []<Affected Package Information>
-func loadParts(partsInputPath string) map[cves.CVEID][]vulns.PackageInfo {
+// A mapping of "CVE-ID": time.Time (the latest modified time of its part files)
+func loadParts(partsInputPath string) (map[cves.CVEID][]vulns.PackageInfo, map[cves.CVEID]time.Time) {
 	dir, err := os.ReadDir(partsInputPath)
 	if err != nil {
 		Logger.Fatalf("Failed to read dir %q: %s", partsInputPath, err)
 	}
 	output := map[cves.CVEID][]vulns.PackageInfo{}
+	cvePartsModifiedTime := make(map[cves.CVEID]time.Time)
 	for _, entry := range dir {
 		if !entry.IsDir() {
 			Logger.Warnf("Unexpected file entry %q in %s", entry.Name(), partsInputPath)
 			continue
 		}
 		// map is already a reference type, so no need to pass in a pointer
-		loadInnerParts(path.Join(partsInputPath, entry.Name()), output)
+		loadInnerParts(path.Join(partsInputPath, entry.Name()), output, cvePartsModifiedTime)
 	}
-	return output
+	return output, cvePartsModifiedTime
 }
 
 // combineIntoOSV creates OSV entry by combining loaded CVEs from NVD and PackageInfo information from security advisories.
-func combineIntoOSV(loadedCves map[cves.CVEID]cves.Vulnerability, allParts map[cves.CVEID][]vulns.PackageInfo, cveList string) map[cves.CVEID]*vulns.Vulnerability {
+func combineIntoOSV(loadedCves map[cves.CVEID]cves.Vulnerability, allParts map[cves.CVEID][]vulns.PackageInfo, cveList string, cvePartsModifiedTime map[cves.CVEID]time.Time) map[cves.CVEID]*vulns.Vulnerability {
 	Logger.Infof("Begin writing OSV files")
 	convertedCves := map[cves.CVEID]*vulns.Vulnerability{}
 	for cveId, cve := range loadedCves {
@@ -129,6 +162,10 @@ func combineIntoOSV(loadedCves map[cves.CVEID]cves.Vulnerability, allParts map[c
 		for _, pkgInfo := range allParts[cveId] {
 			convertedCve.AddPkgInfo(pkgInfo)
 		}
+		cveModified, _ := time.Parse(time.RFC3339, convertedCve.Modified)
+		if cvePartsModifiedTime[cveId].After(cveModified) {
+			convertedCve.Modified = cvePartsModifiedTime[cveId].Format(time.RFC3339)
+		}
 		convertedCves[cveId] = convertedCve
 	}
 	return convertedCves
@@ -137,7 +174,7 @@ func combineIntoOSV(loadedCves map[cves.CVEID]cves.Vulnerability, allParts map[c
 // writeOSVFile writes out the given osv objects into individual json files
 func writeOSVFile(osvData map[cves.CVEID]*vulns.Vulnerability, osvOutputPath string) {
 	for vId, osv := range osvData {
-		file, err := os.OpenFile(path.Join(osvOutputPath, string(vId) + ".json"), os.O_CREATE|os.O_RDWR, 0644)
+		file, err := os.OpenFile(path.Join(osvOutputPath, string(vId)+".json"), os.O_CREATE|os.O_RDWR, 0644)
 		if err != nil {
 			Logger.Fatalf("Failed to create/open file to write: %s", err)
 		}

vulnfeeds/cmd/combine-to-osv/main_test.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"testing"
+	"time"
 
 	"golang.org/x/exp/maps"
 
@@ -34,7 +35,7 @@ func loadTestData2(cveName string) cves.Vulnerability {
 }
 
 func TestLoadParts(t *testing.T) {
-	allParts := loadParts("../../test_data/parts")
+	allParts, _ := loadParts("../../test_data/parts")
 	expectedPartCount := 14
 	actualPartCount := len(allParts)
 
@@ -87,9 +88,9 @@ func TestCombineIntoOSV(t *testing.T) {
 		"CVE-2022-32746":   loadTestData2("CVE-2022-32746"),
 		"CVE-2018-1000500": loadTestData2("CVE-2018-1000500"),
 	}
-	allParts := loadParts("../../test_data/parts")
+	allParts, cveModifiedTime := loadParts("../../test_data/parts")
 
-	combinedOSV := combineIntoOSV(cveStuff, allParts, "")
+	combinedOSV := combineIntoOSV(cveStuff, allParts, "", cveModifiedTime)
 
 	expectedCombined := 3
 	actualCombined := len(combinedOSV)
@@ -103,3 +104,50 @@ func TestCombineIntoOSV(t *testing.T) {
 		}
 	}
 }
+
+func TestGetModifiedTime(t *testing.T) {
+	_, err := getModifiedTime("../../test_data/parts/debian/CVE-2016-1585.debian.json")
+	if err != nil {
+		t.Errorf("Failed to get modified time.")
+	}
+
+}
+
+func TestUpdateModifiedDate(t *testing.T) {
+	var cveId1, cveId2 cves.CVEID
+	cveId1 = "CVE-2022-33745"
+	cveId2 = "CVE-2022-32746"
+
+	cveStuff := map[cves.CVEID]cves.Vulnerability{
+		cveId1: loadTestData2("CVE-2022-33745"),
+		cveId2: loadTestData2("CVE-2022-32746"),
+	}
+	allParts, _ := loadParts("../../test_data/parts")
+
+	cveModifiedTimeMock := make(map[cves.CVEID]time.Time)
+	time1 := "0001-00-00T00:00:00Z"
+	time2 := "2024-04-30T00:38:53Z"
+	modifiedTime1, _ := time.Parse(time.RFC3339, time1)
+	modifiedTime2, _ := time.Parse(time.RFC3339, time2)
+	cveModifiedTimeMock[cveId1] = modifiedTime1
+	cveModifiedTimeMock[cveId2] = modifiedTime2
+
+	combinedOSV := combineIntoOSV(cveStuff, allParts, "", cveModifiedTimeMock)
+
+	expectedCombined := 2
+	actualCombined := len(combinedOSV)
+
+	if actualCombined != expectedCombined {
+		t.Errorf("Expected %d in combination, got %d: %#v", expectedCombined, actualCombined, combinedOSV)
+	}
+
+	// Keeps CVE modified time if none of its parts have a later modification time
+	if combinedOSV[cveId1].Modified == time1 {
+		t.Errorf("Wrong modified time: %s", combinedOSV["CVE-2022-33745"].Modified)
+	}
+
+	// Updates the CVE's modified time if any of its parts have a later modification time
+	if combinedOSV[cveId2].Modified != time2 {
+		t.Errorf("Wrong modified time, expected: %s, got: %s", time2, combinedOSV["CVE-2022-32746"].Modified)
+	}
+}

vulnfeeds/cmd/debian/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/csv"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"os"
 	"path"
@@ -59,6 +60,10 @@ func getDebianReleaseMap() (map[string]string, error) {
 	}
 	defer res.Body.Close()
 
+	if res.StatusCode != http.StatusOK {
+		return releaseMap, fmt.Errorf("HTTP request failed: %s", res.Status)
+	}
+
 	reader := csv.NewReader(res.Body)
 	reader.FieldsPerRecord = -1
 	data, err := reader.ReadAll()
@@ -198,6 +203,12 @@ func downloadDebianSecurityTracker() (DebianSecurityTrackerData, error) {
 		return nil, err
 	}
 
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP request failed: %s", res.Status)
+	}
+
 	var decodedDebianData DebianSecurityTrackerData
 
 	if err := json.NewDecoder(res.Body).Decode(&decodedDebianData); err != nil {

vulnfeeds/cmd/debian/main_test.go

@@ -18,12 +18,12 @@ func Test_generateDebianSecurityTrackerOSV(t *testing.T) {
 	_ = json.NewDecoder(file).Decode(&decodedDebianData)
 
 	debianReleaseMap := make(map[string]string)
-	debianReleaseMap["trixie"] = "13"
+	debianReleaseMap["sarge"] = "3.1"
+	debianReleaseMap["stretch"] = "9"
 	debianReleaseMap["buster"] = "10"
 	debianReleaseMap["bullseye"] = "11"
 	debianReleaseMap["bookworm"] = "12"
-	debianReleaseMap["sarge"] = "3.1"
-	debianReleaseMap["stretch"] = "9"
+	debianReleaseMap["trixie"] = "13"
 
 	osvPkgInfos := generateDebianSecurityTrackerOSV(decodedDebianData, debianReleaseMap)
 	expectedCount := 2