From a9e57a2f8ecb77a1ba8b1f4bdc016cd1e0412f96 Mon Sep 17 00:00:00 2001 From: Willem de Groot Date: Sun, 2 Apr 2023 13:05:11 +0200 Subject: [PATCH 1/5] Add auto-update --- corediff.go | 12 +++++++++++- go.mod | 22 +++++++++++----------- go.sum | 48 +++++++++++++++++++++++++++++------------------- 3 files changed, 51 insertions(+), 31 deletions(-) diff --git a/corediff.go b/corediff.go index e8d9843..49ed368 100644 --- a/corediff.go +++ b/corediff.go @@ -8,9 +8,15 @@ import ( "log" "os" "path/filepath" + "runtime" + + "github.com/gwillem/go-selfupdate" ) -var placeholder = struct{}{} +var ( + selfUpdateURL = fmt.Sprintf("https://sansec.io/downloads/%s-%s/corediff", runtime.GOOS, runtime.GOARCH) + placeholder = struct{}{} +) func loadDB(path string) hashDB { m := make(hashDB) @@ -187,6 +193,10 @@ func addPath(root string, db hashDB, args *baseArgs) { func main() { + if restarted, err := selfupdate.UpdateRestart(selfUpdateURL); restarted || err != nil { + logVerbose("Restarted new version", restarted, "with error:", err) + } + args := setup() db := loadDB(args.Database) diff --git a/go.mod b/go.mod index f08a892..ee6c8fb 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,22 @@ -module github.com/gwillem/magento-corediff +module github.com/sansecio/corediff go 1.20 require ( - github.com/cespare/xxhash/v2 v2.2.0 - github.com/fatih/color v1.12.0 + github.com/fatih/color v1.15.0 github.com/gobwas/glob v0.2.3 - github.com/gwillem/urlfilecache v0.0.0-20201121123616-8e4f7e58333d + github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292 + github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13 github.com/jessevdk/go-flags v1.5.0 - github.com/stretchr/testify v1.6.1 + github.com/stretchr/testify v1.8.2 ) require ( - github.com/adrg/xdg v0.2.3 // indirect - github.com/davecgh/go-spew v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect + github.com/adrg/xdg v0.4.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + golang.org/x/sys v0.6.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e96fc2f..081a524 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,41 @@ -github.com/adrg/xdg v0.2.3 h1:GxXngdYxNDkoUvZXjNJGwqZxWXi43MKbOOlA/00qZi4= -github.com/adrg/xdg v0.2.3/go.mod h1:7I2hH/IT30IsupOpKZ5ue7/qNi3CoKzD6tL3HwpaRMQ= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/gwillem/urlfilecache v0.0.0-20201121123616-8e4f7e58333d h1:SUYp/bOtCvpddj20LJRLoU/YFnoQhXLGG3S+Ro8uWP0= -github.com/gwillem/urlfilecache v0.0.0-20201121123616-8e4f7e58333d/go.mod h1:wcspENTUwvMi317k0q22DWfx6hIXAlEjz5LO6pjJWQ0= +github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292 h1:AoF/jIYBBxpLJkyf204k6ArAFDFVKitkcOGnyYuWHco= +github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292/go.mod h1:Zv9vnDD/HDN/WCidxG2QdCEDtYIaIeUTX9yHspWZfEM= +github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13 h1:Kuu6BZRS1uaSatweU/S5pWKkwt+QFCMb2QjJrWcvTWA= +github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13/go.mod h1:vEuNDp+GXXq7N5Anwiu1f4VQ08IGuSTGs/PaFCEno/U= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 11f5bd189899bfc4295ef10228f8a45edfb538a8 Mon Sep 17 00:00:00 2001 From: Willem de Groot Date: Sun, 2 Apr 2023 14:19:29 +0200 Subject: [PATCH 2/5] Add auto-updater and version string --- corediff.go | 8 +++++--- go.mod | 5 ++++- go.sum | 8 ++++++-- needle_test.go | 26 ++++++++++++++++++++++++++ setup.go | 2 +- 5 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 needle_test.go diff --git a/corediff.go b/corediff.go index 49ed368..54326b2 100644 --- a/corediff.go +++ b/corediff.go @@ -10,12 +10,14 @@ import ( "path/filepath" "runtime" + "github.com/gwillem/go-buildversion" "github.com/gwillem/go-selfupdate" ) var ( - selfUpdateURL = fmt.Sprintf("https://sansec.io/downloads/%s-%s/corediff", runtime.GOOS, runtime.GOARCH) - placeholder = struct{}{} + selfUpdateURL = fmt.Sprintf("https://sansec.io/downloads/%s-%s/corediff", runtime.GOOS, runtime.GOARCH) + placeholder = struct{}{} + corediffVersion = buildversion.String() ) func loadDB(path string) hashDB { @@ -200,7 +202,7 @@ func main() { args := setup() db := loadDB(args.Database) - logInfo(boldwhite("Corediff loaded ", len(db), " precomputed hashes. (C) 2020-2023 labs@sansec.io")) + logInfo(boldwhite("Corediff ", corediffVersion, " loaded ", len(db), " precomputed hashes. (C) 2020-2023 labs@sansec.io")) logInfo("Using database:", args.Database, "\n") if args.Merge { diff --git a/go.mod b/go.mod index ee6c8fb..cd23334 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/fatih/color v1.15.0 github.com/gobwas/glob v0.2.3 + github.com/gwillem/go-buildversion v0.0.0-20230402114832-b1addefa8764 github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292 github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13 github.com/jessevdk/go-flags v1.5.0 @@ -15,8 +16,10 @@ require ( github.com/adrg/xdg v0.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/sys v0.6.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +//replace github.com/gwillem/go-buildversion => ../go-buildversion diff --git a/go.sum b/go.sum index 081a524..bbff535 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,10 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gwillem/go-buildversion v0.0.0-20230402113215-08d191152cc5 h1:tMpRA020WZ+HsqrRFiflnC0Pj9gseMrG8jWimXw9WWQ= +github.com/gwillem/go-buildversion v0.0.0-20230402113215-08d191152cc5/go.mod h1:mT3drjAmZ/WgzCwPqf270xC9absLnjSMbDSPXPRiGk8= +github.com/gwillem/go-buildversion v0.0.0-20230402114832-b1addefa8764 h1:GL/gS9rv5F1oBgQwbT+pl0HefNbDZLjbcWmDNT/pUOU= +github.com/gwillem/go-buildversion v0.0.0-20230402114832-b1addefa8764/go.mod h1:yf2A6rXn1ptErQTutA0vpEDZJG6E0b/aUnMnXelY5tU= github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292 h1:AoF/jIYBBxpLJkyf204k6ArAFDFVKitkcOGnyYuWHco= github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292/go.mod h1:Zv9vnDD/HDN/WCidxG2QdCEDtYIaIeUTX9yHspWZfEM= github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13 h1:Kuu6BZRS1uaSatweU/S5pWKkwt+QFCMb2QjJrWcvTWA= @@ -16,8 +20,8 @@ github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/needle_test.go b/needle_test.go new file mode 100644 index 0000000..6bbd387 --- /dev/null +++ b/needle_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "fmt" + "hash/crc32" + "testing" +) + +func Test_Needle(t *testing.T) { + + needles := []string{ + "path:pub/static/frontend/BodyAndBeach/store/fr_CA/Altima_Lookbookslider/js/jquery.cycle2_1.js", + "path:app/code/Magedelight/GeoIp/Controller/Adminhtml/Currencymapping/Delete.php", + } + dbpath := "m2.db" + + db := loadDB(dbpath) + fmt.Println("Loaded entries:", len(db)) + + for _, needle := range needles { + checksum := crc32.ChecksumIEEE([]byte(needle)) + hash := fmt.Sprintf("%x", checksum) + _, ok := db[checksum] + fmt.Println(ok, hash) + } +} diff --git a/setup.go b/setup.go index 3ede7d9..f1b6fd5 100644 --- a/setup.go +++ b/setup.go @@ -121,7 +121,7 @@ func setup() *baseArgs { fmt.Println("Can't merge without given --database file") os.Exit(1) } - fmt.Println("Downloading default hash database from", hashDBURL) + // fmt.Println("Using default hash database from", hashDBURL) args.Database = urlfilecache.ToPath(hashDBURL) } From 2a8b368ac442b26648fddcca98d84a82f8e3c655 Mon Sep 17 00:00:00 2001 From: Willem de Groot Date: Sun, 2 Apr 2023 14:31:54 +0200 Subject: [PATCH 3/5] Switch to xxhash because of collissions --- corediff.go | 2 +- corediff_test.go | 9 ++++++--- fixture/sample.db | Bin 952 -> 1904 bytes go.mod | 1 + go.sum | 2 ++ helpers.go | 8 ++++---- needle_test.go | 8 ++++++-- setup.go | 2 +- 8 files changed, 21 insertions(+), 11 deletions(-) diff --git a/corediff.go b/corediff.go index 54326b2..e69a292 100644 --- a/corediff.go +++ b/corediff.go @@ -31,7 +31,7 @@ func loadDB(path string) hashDB { defer f.Close() reader := bufio.NewReader(f) for { - var b uint32 + var b uint64 err = binary.Read(reader, binary.LittleEndian, &b) if err == io.EOF { break diff --git a/corediff_test.go b/corediff_test.go index 7f4ee91..52d8551 100644 --- a/corediff_test.go +++ b/corediff_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -func digest(b uint32) string { +func digest(b uint64) string { return fmt.Sprintf("%x", b) } @@ -28,7 +28,7 @@ func Test_hash(t *testing.T) { args []byte want string }{ - {[]byte("banaan"), "14ac6691"}, + {[]byte("banaan"), "acfb1ff4438e39f3"}, } for _, tt := range tests { t.Run(string(tt.args), func(t *testing.T) { @@ -42,8 +42,11 @@ func Test_hash(t *testing.T) { func Test_vendor_bug(t *testing.T) { db := loadDB("fixture/sample.db") assert.Len(t, db, 238) - wantHash := uint32(3333369281) + wantHash := uint64(3900178074848893275) if _, ok := db[wantHash]; !ok { + for k, _ := range db { + fmt.Println(k) + } t.Error("hash not in db") } } diff --git a/fixture/sample.db b/fixture/sample.db index d395b42074415c02b1851752c83ee4972e341f3f..b216a96af874ecbd8f634242e932e643c3eed8af 100644 GIT binary patch literal 1904 zcmV-$2aovjts)n=wKzcs7_ue!+f0N0JVsJhH1=V1Tq&H~= z&IGEgsrywD-bInC_Vcx7bfZ03rly|DZNa-fZV7c-D7oW>xj;nzZ>x+{`r) zb?Fdtaym=aEBf^H;5f?0?Ea(g^Nl7(OX(imb|snw0s05~N(}s-e$T1*gL9c~QPHLd z1U%-^HW0Kb-X*D&R)5*hK~hyHKr(0;bxOfQjznLCbk(pio1ztt3CLWS0GhdQan>nr zzawkSAxYff?l7Kown=|WRC0`5`5J4aXdrh)J`#w0je+y7%nlV|E@24Atl*H5M-{7{ zaNTq(l%SX=umC(O)7S!ig3kCUSO#6-Ay?i`4=k(`vkKQ&M8P~{j;^HB|F6?)<3X|6 z^ci-a)UAO4dpNh7M&9INITB>I`dGQ`6{RBNXw~!%2Hq)VVwgcSN-p#>v_b`?&_yG& zuV?o%doY%^PRSF!$vIol!4`T6>ke75Tekw&6G#Zo>iu{n`Wpc3`9kPj=vzq2S`a*m zg7_bMo(6)(ogZqzew#Qo`~Oemix@v-@@F5KskZra$^ynIecwHy}2)4yS7zVrUHXG;pX@I1$z%oBy#z(ea@- z-Q>CFaNJc~&%a89UzEJhtT`%eDq zg|}!saAk6&iDuifiR3f);|Fg5K9v+6jT&}FqChzPrE29{j1~h!faS!X3PQ|Wy#|kc ziA$WtXkkV+hAoEFr*Fl9VfsA?Y6CyaPM0S^m3f~Zm#UTQ<(qnQrHh`yL5X-G7<48K z)@3b5wjhb;K07&>rX|sCg)xxDVo6x?AN6-IK90V!^M)taLLE_S$W}7}mwmb{DE-{Y zNX}BVHC$3fc*CLUO5-dRwqTL0n~)^JJJlGSfuCuyzc?`BFnIH0%W7{#%+oDDW;>+eYu42N4ld z!HQ32@5PRkW)=6)ObJV9_bYB<Q*%2qD;TYd9 zj6Yw+8xHt@v$}RbWN$Z8Sp~?R8fTcK8ui^?;~|bkHV2LZ*J`@F@)74Pz7QI=%hNk2 zSBl~6Ty78B92-7awn=IvflNL5ks-5P*!V8>%MqS4pRUSNSpk`CbN+a!yF(X_+R6sA6LIOWOrS)mq;2~Ej*s*1 zK6k)#(}Z3JiyeS1(^pXz#bqKPSOqVIphd?c`$^D7^p%&Vi;%aauE>lw$$jcwScoOn z>}NIJWDf|UrYAj1odYaY8%UTPu7?A0&9WBkGE?)rAI1pdHAK6Fdk!~+o!#*0*MLK( z1b1nUdjGW#p1*huqru3Q*u`97Y2PSvj9aLChZhbwt(mdPX@3u4($4x(LaQC)Y0Kn8WM+G|D;Q5Q|BU-(8@@s?RJtNR0j zR8kl_D?hd1L^^Wy+Bl35LJK$`d26pd_u2>A`kDqG6G5Ua+4k0~>!RWzeX}iwNmt)h)m9eBF7$FOL(km-%wcU z@C=Iz(e}9~;yElkXtcoBmd2+1{weA~t+^5FlB*&C!Izn6g>R=&-9BA#hI+b6;u%aJ z4TsLy$@73-apBgQQ_J}c*H$xa{~kWxs9;oeTMuFc@D%)bX>#L^_P;h4D$UJeEe$N) qy9wK#Aow57S-@UC+$ebX0p9vD#q{A0g_!^jM10pZiQ~bvP_p{)5UeNw literal 952 zcmV;p14sO&@)OxKIPRB|3?0k3bVlf?g%y{%>@9Rz@xFD94W3^S$&N%<&ysaEpi+8q z9G;NA?DYJAM~ne@2P7~WoDBY8{6lKpra?Jy!3G-`wqXhog#im1+TY0W9sHgu-zNUS zEw9Fw6R;wOY}M)l1b>lWrUD2tS|O>*pUFVd{MjYteAvR4xqhOeXcp0XWa|cH1So&X zVS#^psx-;=0rr=l*Eci@OX~&FPOzekTCA|JD07==oy5tE>T7^+E&S8ExcJ z^#chwkSkvn^fZDxNY`g;=R$rw^`q-NVM(59M<}>Su^#BIvTLK@CvA`tXGYJq0gk7V z?2J74{vG?*+?R#M9QP{wHbt&}-{foZ=R=YJ0000C;SSQf2N;3u0u!HI;viFVn;yjV3SHZDY;JNebpH zdJf#u1g0G#YG67*)41;=VBc_IzG}7%;@}qIjQGFv@X!-4GEup;=g-CYoZPv|ZkB=* zBSzjSpGeJNF0mW<6g)|z%#Y~xu>hDE0E@Z(F zyj)8Ve?X9 zgEh;TC7=z+X5VNBg%R`kx(Mmd$rPjPceT-DBSk?`{l-!2gLh0Qwtv;PMqSMQnWnFO zu>uhRJ8op_bSt8r;1z4S>rDiO+xo;QRqdQ>*uNB8bo3C}Yb9xL2j;IDWuw4rnZYFQ8B-F=5wo#K4h(cAmyHy9#lbd)y3R6#20N22WxaC1Xr?csB7s-Lzm z8bGt`vw*;?7G#r>?Ye3U@dvSN`W%U9% aUZjm=XK7dy_qH^kt{?m?m#2LgUMfl)kk&E) diff --git a/go.mod b/go.mod index cd23334..17ed5e8 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( require ( github.com/adrg/xdg v0.4.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.18 // indirect diff --git a/go.sum b/go.sum index bbff535..dd25a70 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/helpers.go b/helpers.go index 99a91d6..192a91f 100644 --- a/helpers.go +++ b/helpers.go @@ -3,12 +3,12 @@ package main import ( "bytes" "fmt" - "hash/crc32" "os" "path/filepath" "regexp" "strings" + "github.com/cespare/xxhash/v2" "github.com/gobwas/glob" ) @@ -78,11 +78,11 @@ func check(err error) { } } -func hash(b []byte) uint32 { - return crc32.ChecksumIEEE(b) +func hash(b []byte) uint64 { + return xxhash.Sum64(b) } -func pathHash(p string) uint32 { +func pathHash(p string) uint64 { return hash([]byte("path:" + p)) } diff --git a/needle_test.go b/needle_test.go index 6bbd387..4a4aae7 100644 --- a/needle_test.go +++ b/needle_test.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "hash/crc32" "testing" ) @@ -17,8 +16,13 @@ func Test_Needle(t *testing.T) { db := loadDB(dbpath) fmt.Println("Loaded entries:", len(db)) + for k, _ := range db { + fmt.Println("first entry", k) + break + } + for _, needle := range needles { - checksum := crc32.ChecksumIEEE([]byte(needle)) + checksum := hash([]byte(needle)) hash := fmt.Sprintf("%x", checksum) _, ok := db[checksum] fmt.Println(ok, hash) diff --git a/setup.go b/setup.go index f1b6fd5..8e7995f 100644 --- a/setup.go +++ b/setup.go @@ -11,7 +11,7 @@ import ( ) type ( - hashDB map[uint32]struct{} + hashDB map[uint64]struct{} walkStats struct { totalFiles int From 0452a1213fb309c04243e2aa54bb6c79194e04ec Mon Sep 17 00:00:00 2001 From: Willem de Groot Date: Sun, 2 Apr 2023 14:43:29 +0200 Subject: [PATCH 4/5] Remove verbose logging in tests --- corediff_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/corediff_test.go b/corediff_test.go index 52d8551..8af8390 100644 --- a/corediff_test.go +++ b/corediff_test.go @@ -44,9 +44,6 @@ func Test_vendor_bug(t *testing.T) { assert.Len(t, db, 238) wantHash := uint64(3900178074848893275) if _, ok := db[wantHash]; !ok { - for k, _ := range db { - fmt.Println(k) - } t.Error("hash not in db") } } From 7fbf53d7a8098e765d89c3db59dfea22578528f8 Mon Sep 17 00:00:00 2001 From: Willem de Groot Date: Sun, 2 Apr 2023 19:43:42 +0200 Subject: [PATCH 5/5] Remove specific test --- needle_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/needle_test.go b/needle_test.go index 4a4aae7..09006ba 100644 --- a/needle_test.go +++ b/needle_test.go @@ -8,7 +8,6 @@ import ( func Test_Needle(t *testing.T) { needles := []string{ - "path:pub/static/frontend/BodyAndBeach/store/fr_CA/Altima_Lookbookslider/js/jquery.cycle2_1.js", "path:app/code/Magedelight/GeoIp/Controller/Adminhtml/Currencymapping/Delete.php", } dbpath := "m2.db"