From 68eabfa5d1fbaf7454e1697f0d3f6a82c710482c Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 13 Oct 2023 12:58:53 -0700 Subject: [PATCH] s2: Fix S2 best dictionary wrong encoding (#871) A dictionary match one byte out of dictionary range was possible. Recovery is possible. Send me a request if you find any content with this error in the [discussion](https://github.com/klauspost/compress/discussions/categories/general). --- s2/dict_test.go | 3 ++- s2/encode_best.go | 3 +++ s2/testdata/fuzz/dict-corpus-oss.zip | Bin 0 -> 7664 bytes 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 s2/testdata/fuzz/dict-corpus-oss.zip diff --git a/s2/dict_test.go b/s2/dict_test.go index 1b478e4165..9ac9f70c7c 100644 --- a/s2/dict_test.go +++ b/s2/dict_test.go @@ -402,13 +402,14 @@ func TestDictSize(t *testing.T) { } }) } - fmt.Printf("%d files, %d -> %d (%.2f%%) - %.02f bytes saved/file\n", totalCount, totalIn, totalOut, float64(totalOut*100)/float64(totalIn), float64(totalIn-totalOut)/float64(totalCount)) + t.Logf("%d files, %d -> %d (%.2f%%) - %.02f bytes saved/file\n", totalCount, totalIn, totalOut, float64(totalOut*100)/float64(totalIn), float64(totalIn-totalOut)/float64(totalCount)) } func FuzzDictBlocks(f *testing.F) { fuzz.AddFromZip(f, "testdata/enc_regressions.zip", fuzz.TypeRaw, false) fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-raw.zip", fuzz.TypeRaw, testing.Short()) fuzz.AddFromZip(f, "testdata/fuzz/block-corpus-enc.zip", fuzz.TypeGoFuzz, testing.Short()) + fuzz.AddFromZip(f, "testdata/fuzz/dict-corpus-oss.zip", fuzz.TypeOSSFuzz, testing.Short()) // Fuzzing tweaks: const ( diff --git a/s2/encode_best.go b/s2/encode_best.go index 1d13e869a1..47bac74234 100644 --- a/s2/encode_best.go +++ b/s2/encode_best.go @@ -157,6 +157,9 @@ func encodeBlockBest(dst, src []byte, dict *Dict) (d int) { return m } matchDict := func(candidate, s int, first uint32, rep bool) match { + if s >= MaxDictSrcOffset { + return match{offset: candidate, s: s} + } // Calculate offset as if in continuous array with s offset := -len(dict.dict) + candidate if best.length != 0 && best.s-best.offset == s-offset && !rep { diff --git a/s2/testdata/fuzz/dict-corpus-oss.zip b/s2/testdata/fuzz/dict-corpus-oss.zip new file mode 100644 index 0000000000000000000000000000000000000000..99a06c13f252bddcf165dff9717559566b33b140 GIT binary patch literal 7664 zcmcJUbyQSqyTAtq>5veR24QFI+)lvvxC7rTws2F zZeAWfK0X0nPGdvU{#(c#dRBUqqqU&Cyi;wO51he@Uj`S^l*eemqW6G6&D@Am4&Za- z;5Jk`O@B?QVo6;MK>~U@w4noPbeG=swKXo7|Ix?E>$@WNEoY2iuxs&J6nauyL!x^? zqBoo_$erxBaqoA$jLRo5B_%z%s|W*g2R%0$Wd4Zq;fWZig$6`_tG1ga2CG|tOwDk) z-r*c8>JimzRAL%%IVYby_Kvx^KQk~IaI|GC6Ju7_0EH6@xM-Q5C2 z9|CWKjLpr>NwKEH1Zvd_KUkk5YT zT%cZh1btL2l*;Uft|1sV^a}Z1QJV2?-$W1Bc55yF$S1ZTy$ze9ODti*?G@PX8l<(I zN|?nSwvN?XC);i_MLHQMRkXOOC0p+tb{vu&VmpGG=@W_IZ8?b>I!=3iPoZwiU(73_ z5u8e(_SQrcxT!LP7;fIOS$h+_yb1Ddg7}+&!zk;T_MbxE-&|pDlm$tvJU{zrH+6Y( zX;U>lwX|~r?GB>UcNfqdODmfGrL1n@Rn5N7+SSEi3g!6e<-0Ks_kqeE7e74|&yT{1sBIcjqP`ya`BdfXUl80nttHSA_lfmPrOkYzWui z2A`;6byo^KSc_5~jBRMxaQ<-6wWnpzExgZ>WIYNN+CGwn0w33>Q;YfgMjz%08&_oal_3eQ+ccIFB}Mv@r_8-N67cx0S#b}w_8=dG z_Bqnw(Fyxw&v*IKm%m^M);w47EUziMLVSIgD%-cQ#2={sPfu4w+kW@KZ{~9SKQq?} zyC*L$HgSF9�=1&iOq6_TEgc*ZZUVN3%8umODx_@2O)AcKf17x_FL7_4{ zvu+8$KiRVRKf4|0U-nV8De`RZj$W`+*Baz6;qh`mOU7ToP46apRUU)eDi;!Oc*?1iF(^m*M=0{vxgKl6QHU4w zk4pG!Tcgt2zm!HUuV|j-0QH#O8*P!Kj^g(K*Ytu4y&U%=J&fl9JO>y-qfI_84nu}1 zREVBDQ$rU+TSFg1V?!rHYr~W#0*adsJDu6VHKKWeM} z<@|=v$B^jH>Y#l<`B0 zVQU4^*65z}$>=xnKLZl{$Kpd{dfIC{&EY?%Bb0YxEI?C_uoWP>)T%qN${))+Wi&` z*j^m}wmJbj3^xoX3=e`0!G+*-lOd=O;(h<3fB$~`4Xt=qJfXs`>HglTt%x;L{B^~T z-6~doZdz#?X*y|YZCYsh)>Q1j>VNmZ_V@U)G4wJ-6|Va?1!(`4`pba*&ObzAdr|4$ zzIsx+W$3Io>8Llkn_gZ!pN&WDC9rgf$Is6nd9?%;@Hv~+!~0afI*S;lo?3>UG<_@) zp>P;aY$+Pb;g4beN4zxVunezM>wf@s2pFIQ-XnIfS_oOww$8{4p-UJvofy_<7d=qH>Zlq2Tf|^k6 z9F3sZ!n}=5v-JI3gZMrt_^M~va$@jOO{>FIGxbGtXk)Kz$Olg@^|VaJ#e|I~$EN`g zn>bGy4`)|NPMmIU#$|nJ6U@O?^yO67>D4U?qx3dwPlLPrgg%jMHuhhUw(%+ zZzIx|O|oZ*wmf(DEX)_Ode?0=aq=Rs|Ib|Tym|Cf3=gJ zO?p&X^&eoJzJ^1({@3rJnSI1*&GC0MVOfer(5&mge+zXR>?%NEZ~ts1sFXkTY}#jl znR7T<-hVY{DR5w787KVyQpeD6WGkS`DLdW?Lu_d`^KLN|7!*A;o$b`d~t`j2M(9BM4|-2vJTXf@JpZl>%)N|Wbl2MuQ~em00| zrt#D%mknu2uS&B@m8;pK+3IGbs{0OCRS9U+9$f=|m#ZR|vkBK;?G7T7*+;DRuTQW= zudX5w_V)Is61MA~;a%^|tY2U66$H)m?|i~lIZaJpe|C%}dTn@|@6Ldiw#sO5bsT*; zr{L;)mf3I?iFZaA+28emDlbxW^7zBfUa;szUG6E_MuGh2DY3V%+3(%(1@%5{*;n+o$jpRm6(*)O z7Z%l5jd5ffWgRie%H_6iLI%q#cZ^q-!rpZ-)oAffRO_*f=O{V9*I2G=!^HIox00LW zY_Vz&pCo$#3ZXs1C4I;{Ih4h~$=KJ+G`SUQ?!Ra*%Y0&Pe8w#my}36aUnrc*56@WK z94kz!JGATc4V-@pv9xT3-+CjPw$~<6U920ga-dvbo8U5KKPlCQ(O0u1&?hn%kA1j# zi0l1wDRY%}z?MSi4zswGGT)f3k~_Z~8@0~A@Rs^+ebSk;48dX?L zrPUahzzwamZq&mZDsG&D&WU2ZUfao5S38O%rB|9bF(Jv*tjKoTYWM};5QT5PvI93v zGs4Q5arAMOtg}U=lGUJ-Pg-I5=Se}+<TZSDrmFne`WC2^MU)kRD>@^L{7Yg?Vl z0;`yJ7t(jeZrS@Us_5_+JG%E(dc5&0*CSUerJ(Uv(`eqwV;dkJium$0U{nnF+V`GL zt}R*d{M+HMw}mUM)=J9pbpy-aZs!^YuM&GRGpy1|l`J}8b}zgi4sy$-bJ2KCh}P_) zMqN>73jRREy|(&A-P5wK8c!$4YAWi%lu%6T5x9vvVX?fkD_7ST_nF&Ub_IhVJ5?Xn z`+@`GgGg|bwEr?iNyPK}vb0?A+O&WiQy&*G3)#-T#Rxpc-f~GNjt2@85Knbf*OU2u3M>n+7eX z3G-eP=|m;d7aNnB_@L(~89*9V;QLFlJ{;YTpE-)f?!txufbp65c!$TYUy8ALmlnPv zW!N!T5(Wg98{s@+m*{8;X_1T1{7f#MyEij}7!VK@AX1G=FpU|5VoR&_s1*f(XHCoo&Bg&xAdvP#Tl1+DO`wDm;k0 zbrd%m;2sTX9If{VvMWqbvHdoHcE!mbvfoKuT~D_gpvCNi)Si?xpfPU{X^76L zK8P6cPxKlF15{9o_W@&K7%)KWdnDM>L~90`M)zt9D!_I@tUWRO!2}y1az-=F_A`Dw zi~xY`h!YRctybW`Z?H2~CwkY!MPopefdrnx1<(Uqn*zMaqW}VudZj*5Qmt@gh9_p&nKuF{($Eh}v8E7T zgXQgAZg!aNHc`_7fq*mSEWm_H)iHuc5=N1cLJ=#o&)}_y8o=IonB`z@40v z<#7rGU>%Ard8-)sJv$*5R_YODeft;VHh{%V!Cs;^S$k4a&ehDI% z)DNt)`ov^mJZ3tJW-Q6$yVI0I(Q#qDH1GvmdA?YU*x|>fO3JLNsS5pBN=x%BRvZqS z2XpE!KE84J9b6J-JI_AG8?hM{c$g=~UlKi;l~p#x?3g?H>(pj$p7{rv8?T+!A?1d=N`T|(`M z>8T#+^j4F55Q{rF@9uTijul*B^bg>~H(VZ(H?_-ePPKP)nW`h`Zh5n;tRSaVDx(I; zP;ne`GNa(pTXPLb26Ef#3Tnpyh8g%?!(*u=_ya8YGs%05jk+UT!3-?#5<)}`^55f- z&)Ka#-*3!G7&kB&#C>0MT=d{wAN6qK-lxsby|2$UNo;oS9AQ1{)ZBgjmG7+a$kzXi zjjZ-7Gz%u@LEK2P;#cJ4;~^vZMz1?}k5?uGAL%GbKr+J*Lmc<9ZumznTUO_huXDTD zPI4qC8E;`F;Ba!Qi0I_Z2%*6IQwq6-9XA$*ff9&6DqrHmXJ6_8jAA3o%FjdeEI{EC zP$zhcI6HXk{iFm=e=(h|1}#aBYKU#3_j_b5*fI6YrK1p9S!f%(2Ns9E_#G>VxH5i1 zz*;P$E1nLgrcbO^Cl*p4Z`?pa>fna~9|j5%Tr%%LkODEh5<`iskN0u{g)JrLXwt zc-H~mIPPKNW&~%c)L>&;D8=TyxagZ+d(_#E9!gs#(AFdWMFJzgKai!B)-ENMvG*ae zwYxJ{+#HL$SbGAF8O{0t`!??)T~%#Ci(U^ zR`mUOC5A;B{!`t6FjW%QDt;F5u+8?G5I6s*3meU=j7S`uqgFGQ>K7LH!!(}F$ewAO zbQ({01{B{8ZKEX6gudg***YD1Dzq%R;AOCLa$X@( zzJ5ZOA-421bxv)or+2Vy>6zgtKGQ*o7D=LSi8hw2UDsJQCy5z=yR~g0PP~Vk40aK1 zL@CQ?xH>Q_4$V}Yoq2-ta7Ft18Xq(lo*5+m#Hg#Zg{7ep)t!{W-7KFT2I;MDLTGo= zk`4ApMZU1UN$Qe0W7}oSf~*_80+omlaLZjJ(d2%78;8fQ%=pf`3+laix?GlAkPmp^ z;2;6I+-d7AA%v|BS>f7K=c*f?qO+}2wz9v(4%WR4yRO`EZaS+*q8;$6HA{?H5f^v4 z4at2gwI=ZPnRDlyjTBqxvwq)($*fSvf!4keo79iZ=k7em)IOTMiDfc!PZU({&u9_j zt#a{jD8{yZRm4gJSy!cVS0qf#>&|?giR*ltf$;`!CN4PCI7Ag<<8nHldkx#Q#Jch! zTV9-)|NLNCn1uWJ(~Lz3PCyYgG-70jroIQscsZYuvm>b>UU1cR08m{sc3tB16 zWYKXUtpuHZQQ->VagUi?l`>weaqIS03kT@lCyI%I)jNLe1HS1oBCM&O&J4!-Ho#M0 zjnv>A-Y+jLl|xsxCa%W6y|60xF`4Vty1MJO3?_oE$?|e_x`)W`c>T%%u6;3gJ+w8B zok8~!3EGLyCe~xo8P6Wx6cZ+<@1N41d2jhdTH~6esOxIJ`~IcviHrNERSWJu$v)Hc zPGLe(_`~Rjdc-Gl^T#6B8-oVer|a7nTTdq)LN6{A8~5fk?q9q*aQ?WIxRSV07VMui zBn8nZ36|z1nFGxjoqG%)HOUMFaoF|sjxD%C&fnv2udr-wIzg8y4LpyDhcwpey5W?F{n@(I{u89#cSN_+U)DLs3`$gy}fkYPYfq%a}4Sw7hV=Sbr#(zwiN9 zp&ijlND*3TbbhSWOC1uMym!S<4Qh}Hje`b*WH;|hKB@)jB-_089l+sm52Sp)0EyU4 z;8Lu%7ECbrj(;#|udiB(CTJ_PO8C@#5yXR8*E>>>o3Pj_s}Wg1g{0k)mjiAq_}l&?Wd8m zQfW%x%BHm8-xyun&5|0Jpd_4JEtw}~B;7d^*i+ja0@IW8gIh6KWqH1-LKU|U(j;=d zieOvH3GL=RrNKvr;76#)@3|*)=he#Om2SHR?jR3KC1L53F*K14_pqJEEXq|52b2&- z&co^}XE|sHIk%?P^?ScryVuq~b^*PmjCS^w*25