Fuzzing Coverage Expansion (#866)

* New fuzzers and corpora were added

fse/fuzz_test.go

@@ -0,0 +1,44 @@
+package fse
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/klauspost/compress/internal/fuzz"
+)
+
+func FuzzCompress(f *testing.F) {
+	fuzz.AddFromZip(f, "testdata/fse_compress.zip", fuzz.TypeRaw, false)
+	f.Fuzz(func(t *testing.T, buf0 []byte) {
+		var s, s2 Scratch
+		b, err := Compress(buf0, &s)
+		if err != nil || b == nil {
+			return
+		}
+		err = s.validateNorm()
+		if err != nil {
+			return
+		}
+		//Decompress
+		got, err := Decompress(b, &s2)
+		if err != nil || len(got) == 0 {
+			return
+		}
+		if !bytes.Equal(buf0, got) {
+			t.Fatal(fmt.Sprintln("FuzzCompress output mismatch\n", len(got), "org: \n", len(buf0)))
+		}
+	})
+}
+
+func FuzzDecompress(f *testing.F) {
+	fuzz.AddFromZip(f, "testdata/fse_decompress.zip", fuzz.TypeRaw, false)
+	f.Fuzz(func(t *testing.T, buf0 []byte) {
+		var s2 Scratch
+		//Decompress
+		got, err := Decompress(buf0, &s2)
+		if err != nil || len(got) == 0 {
+			return
+		}
+	})
+}

huff0/fuzz_test.go

@@ -0,0 +1,90 @@
+package huff0
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/klauspost/compress/internal/fuzz"
+)
+
+func FuzzCompress(f *testing.F) {
+	fuzz.AddFromZip(f, "testdata/fse_compress.zip", fuzz.TypeRaw, false)
+	f.Fuzz(func(t *testing.T, buf0 []byte) {
+		//use of Compress1X
+		var s Scratch
+		if len(buf0) > BlockSizeMax {
+			buf0 = buf0[:BlockSizeMax]
+		}
+		EstimateSizes(buf0, &s)
+		b, re, err := Compress1X(buf0, &s)
+		s.validateTable(s.cTable)
+		s.canUseTable(s.cTable)
+		if err != nil || b == nil {
+			return
+		}
+
+		min := s.minSize(len(buf0))
+
+		if len(s.OutData) < min {
+			t.Errorf("FuzzCompress: output data length (%d) below shannon limit (%d)", len(s.OutData), min)
+		}
+		if len(s.OutTable) == 0 {
+			t.Error("FuzzCompress: got no table definition")
+		}
+		if re {
+			t.Error("FuzzCompress: claimed to have re-used.")
+		}
+		if len(s.OutData) == 0 {
+			t.Error("FuzzCompress: got no data output")
+		}
+
+		dec, remain, err := ReadTable(b, nil)
+
+		//use of Decompress1X
+		out, err := dec.Decompress1X(remain)
+		if err != nil || len(out) == 0 {
+			return
+		}
+		if !bytes.Equal(out, buf0) {
+			t.Fatal(fmt.Sprintln("FuzzCompressX1 output mismatch\n", len(out), "org: \n", len(buf0)))
+		}
+
+		//use of Compress4X
+		s.Reuse = ReusePolicyAllow
+		b, reUsed, err := Compress4X(buf0, &s)
+		if err != nil || b == nil {
+			return
+		}
+		remain = b
+		if !reUsed {
+			dec, remain, err = ReadTable(b, dec)
+			if err != nil {
+				return
+			}
+		}
+		//use of Decompress4X
+		out, err = dec.Decompress4X(remain, len(buf0))
+		if err != nil || out == nil {
+			return
+		}
+		if !bytes.Equal(out, buf0) {
+			t.Fatal(fmt.Sprintln("FuzzCompressX4 output mismatch: ", len(out), ", org: ", len(buf0)))
+		}
+	})
+}
+
+func FuzzDecompress1x(f *testing.F) {
+	fuzz.AddFromZip(f, "testdata/huff0_decompress1x.zip", fuzz.TypeRaw, false)
+	f.Fuzz(func(t *testing.T, buf0 []byte) {
+		var s Scratch
+		_, remain, err := ReadTable(buf0, &s)
+		if err != nil {
+			return
+		}
+		out, err := s.Decompress1X(remain)
+		if err != nil || out == nil {
+			return
+		}
+	})
+}

ossfuzz/cmd/setup_dicts.go

@@ -0,0 +1,100 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"text/template"
+
+	"github.com/klauspost/compress/zip"
+)
+
+var (
+	dictPath   = flag.String("dict-path", "", "dict path")
+	outputFile = flag.String("output-file", "", "output file")
+)
+
+func main() {
+	flag.Parse()
+	if *dictPath == "" {
+		panic("Need a dict path")
+	}
+	if *outputFile == "" {
+		panic("Need an output file")
+	}
+	dicts := getFuzzDicts(*dictPath)
+
+	t, err := template.New("todos").Parse(`
+package zstd
+var fuzzDicts = make([][]byte, 0)
+func init() {
+{{range $val := .}}
+    fuzzDicts = append(fuzzDicts, {{$val}})
+{{end}}
+}
+`)
+	if err != nil {
+		panic(err)
+	}
+	f, err := os.Create(*outputFile)
+	err = t.Execute(f, dicts)
+	if err != nil {
+		panic(err)
+	}
+	f.Close()
+}
+
+func getFuzzDicts(path string) []string {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		panic(err)
+	}
+	var dicts [][]byte
+	for _, tt := range zr.File {
+		if !strings.HasSuffix(tt.Name, ".dict") {
+			continue
+		}
+		func() {
+			r, err := tt.Open()
+			if err != nil {
+				panic(err)
+			}
+			defer r.Close()
+			in, err := io.ReadAll(r)
+			if err != nil {
+				panic(err)
+			}
+			dicts = append(dicts, in)
+		}()
+	}
+	stringDicts := make([]string, 0)
+	for _, d := range dicts {
+		stringedArray := fmt.Sprintf("%v", d)
+		withComma := strings.Replace(stringedArray, " ", ", ", -1)
+		withClosingBracket := strings.Replace(withComma, "]", "}", -1)
+		withOpenBracket := strings.Replace(withClosingBracket, "[", "[]byte{", -1)
+		stringDicts = append(stringDicts, withOpenBracket)
+	}
+	return stringDicts
+}

ossfuzz/ossfuzz.sh

@@ -0,0 +1,96 @@
+#!/bin/bash -eu
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# This script is meant to be run by
+# https://github.com/google/oss-fuzz/tree/master/projects/compress
+
+# In one of the Zstd fuzzers, the "dict" variable is created by reading a series of files.
+# These files are not available at runtime in the OSS-FUzz environment,
+# so we add the contents of these files to a variable and create a new file
+# that is included when we build the fuzzers in OSS-Fuzz.
+mkdir $SRC/setupdicts
+cp $SRC/compress/ossfuzz/cmd/setup_dicts.go $SRC/setupdicts/main.go
+cd $SRC/setupdicts
+go mod init setupdicts
+go mod tidy
+go run main.go --dict-path=$SRC/compress/zstd/testdata/dict-tests-small.zip --output-file=$SRC/compress/zstd/fuzzDicts.go
+cp $SRC/compress/zstd/fuzzDicts.go $OUT/
+# Done creating "dicts" variable.
+
+cd $SRC/compress
+
+# Modify some files. This would be better done upstream.
+sed -i '38 a\
+	if fi == nil { return }' $SRC/compress/internal/fuzz/helpers.go
+printf "package compress\nimport _ \"github.com/AdamKorcz/go-118-fuzz-build/testing\"\n" > registerfuzzdependency.go
+sed -i 's/zr := testCreateZipReader/\/\/zr := testCreateZipReader/g' "${SRC}"/compress/zstd/fuzz_test.go
+sed -i 's/dicts = readDicts(f, zr)/dicts = fuzzDicts/g' "${SRC}"/compress/zstd/fuzz_test.go
+
+if [ "$SANITIZER" != "coverage" ]; then
+	sed -i 's/\"testing\"/\"github.com\/AdamKorcz\/go-118-fuzz-build\/testing\"/g' "${SRC}"/compress/internal/fuzz/helpers.go
+fi
+
+# OSS-Fuzz uses 'go build' to build the fuzzers, so we move the tests
+# we need into scope.
+mv $SRC/compress/zstd/decoder_test.go $SRC/compress/zstd/decoder_test_fuzz.go
+mv $SRC/compress/zstd/zstd_test.go $SRC/compress/zstd/zstd_test_fuzz.go
+mv $SRC/compress/zstd/seqdec_test.go $SRC/compress/zstd/seqdec_test_fuzz.go
+mv $SRC/compress/zstd/dict_test.go $SRC/compress/zstd/dict_test_fuzz.go
+mv $SRC/compress/s2/s2_test.go $SRC/compress/s2/s2_test_fuzz.go
+go mod tidy
+
+# Build fuzzers
+compile_native_go_fuzzer $SRC/compress/flate FuzzEncoding FuzzFlateEncoding
+compile_native_go_fuzzer $SRC/compress/zstd FuzzDecodeAll FuzzDecodeAll
+compile_native_go_fuzzer $SRC/compress/zstd FuzzDecoder FuzzDecoder
+compile_native_go_fuzzer $SRC/compress/zstd FuzzEncoding FuzzZstdEncoding
+compile_native_go_fuzzer $SRC/compress/zstd FuzzDecAllNoBMI2 FuzzDecAllNoBMI2
+compile_native_go_fuzzer $SRC/compress/zstd FuzzNoBMI2Dec FuzzNoBMI2Dec
+compile_native_go_fuzzer $SRC/compress/s2 FuzzDictBlocks FuzzDictBlocks
+compile_native_go_fuzzer $SRC/compress/s2 FuzzEncodingBlocks FuzzEncodingBlocks
+compile_native_go_fuzzer $SRC/compress/zip FuzzReader FuzzReader
+compile_native_go_fuzzer $SRC/compress/snappy/xerial FuzzDecode FuzzDecode
+compile_native_go_fuzzer $SRC/compress/fse FuzzCompress FuzzFSECompress
+compile_native_go_fuzzer $SRC/compress/fse FuzzDecompress FuzzFSEDecompress
+compile_native_go_fuzzer $SRC/compress/huff0 FuzzCompress FuzzHuff0Compress
+compile_native_go_fuzzer $SRC/compress/huff0 FuzzDecompress1x FuzzHuff0Decompress1x
+compile_native_go_fuzzer $SRC/compress/snappy/xerial FuzzEncode FuzzEncode
+
+#Add corpora from compress-fuzz dir
+cp $SRC/compress-fuzz/zstd/compress/fuzz/encode-corpus-raw.zip $OUT/FuzzZstdEncoding_seed_corpus.zip
+cp $SRC/compress-fuzz/zstd/decompress/fuzz/decode-corpus-raw.zip $OUT/FuzzDecodeAll_seed_corpus.zip
+cp $SRC/compress-fuzz/zstd/decompress/fuzz/decode-corpus-raw.zip $OUT/FuzzDecoder_seed_corpus.zip
+cp $SRC/compress-fuzz/zip/fuzz/FuzzReader-raw.zip $OUT/FuzzReader_seed_corpus.zip
+cp $SRC/compress-fuzz/fse/compress/fuzz/fse_compress.zip $OUT/FuzzFSECompress_seed_corpus.zip
+cp $SRC/compress-fuzz/fse/decompress/fuzz/fse_decompress.zip $OUT/FuzzFSEDecompress_seed_corpus.zip
+cp $SRC/compress-fuzz/huff0/compress/fuzz/huff0_compress.zip $OUT/FuzzHuff0Compress_seed_corpus.zip
+cp $SRC/compress-fuzz/huff0/decompress/fuzz/huff0_decompress1x.zip $OUT/FuzzHuff0Decompress1x_seed_corpus.zip
+cp $SRC/compress-fuzz/flate/flate/fuzz/encode-raw-corpus.zip $OUT/FuzzFlateEncoding_seed_corpus.zip
+cp $SRC/compress-fuzz/s2/compress/fuzz/block-corpus-raw.zip $OUT/FuzzDictBlocks_seed_corpus.zip
+cp $SRC/compress-fuzz/s2/compress/fuzz/block-corpus-raw.zip $OUT/FuzzEncodingBlocks_seed_corpus.zip
+cp $SRC/compress-fuzz/snappy/fuzz/FuzzDecode_raw.zip $OUT/FuzzDecode_seed_corpus.zip
+cp $SRC/compress-fuzz/snappy/fuzz/block-corpus-raw.zip $OUT/FuzzEncode_seed_corpus.zip
+
+# Add missing test files to avoid errors and have test files for code coverage
+cp -r $SRC/compress/zstd/testdata $OUT/
+cp -r $SRC/compress/zip/testdata $OUT/
+cp -r $SRC/compress/flate/testdata $OUT/
+cp -r $SRC/compress/s2/testdata $OUT/
+cp -r $SRC/compress/fse/testdata $OUT/
+cp -r $SRC/compress/snappy/xerial/testdata $OUT/
+cp -r $SRC/compress/huff0/testdata $OUT/
+cp -r $SRC/compress/testdata $OUT

s2/fuzz_test.go

@@ -126,5 +126,25 @@ func FuzzEncodingBlocks(f *testing.F) {
 			t.Error(fmt.Errorf("MaxEncodedLen Exceed: input: %d, mel: %d, got %d", len(data), mel, len(comp)))
 			return
 		}
+
+		concat, err := ConcatBlocks(nil, data, []byte{0})
+		if err != nil || concat == nil {
+			return
+		}
+
+		EstimateBlockSize(data)
+		encoded := make([]byte, MaxEncodedLen(len(data)))
+		if len(encoded) < MaxEncodedLen(len(data)) || minNonLiteralBlockSize > len(data) || len(data) > maxBlockSize {
+			return
+		}
+
+		encodeBlockGo(encoded, data)
+		encodeBlockBetterGo(encoded, data)
+		encodeBlockSnappyGo(encoded, data)
+		encodeBlockBetterSnappyGo(encoded, data)
+		dst := encodeGo(encoded, data)
+		if dst == nil {
+			return
+		}
 	})
 }