diff --git a/corediff.go b/corediff.go index e8d9843..e69a292 100644 --- a/corediff.go +++ b/corediff.go @@ -8,9 +8,17 @@ import ( "log" "os" "path/filepath" + "runtime" + + "github.com/gwillem/go-buildversion" + "github.com/gwillem/go-selfupdate" ) -var placeholder = struct{}{} +var ( + selfUpdateURL = fmt.Sprintf("https://sansec.io/downloads/%s-%s/corediff", runtime.GOOS, runtime.GOARCH) + placeholder = struct{}{} + corediffVersion = buildversion.String() +) func loadDB(path string) hashDB { m := make(hashDB) @@ -23,7 +31,7 @@ func loadDB(path string) hashDB { defer f.Close() reader := bufio.NewReader(f) for { - var b uint32 + var b uint64 err = binary.Read(reader, binary.LittleEndian, &b) if err == io.EOF { break @@ -187,10 +195,14 @@ func addPath(root string, db hashDB, args *baseArgs) { func main() { + if restarted, err := selfupdate.UpdateRestart(selfUpdateURL); restarted || err != nil { + logVerbose("Restarted new version", restarted, "with error:", err) + } + args := setup() db := loadDB(args.Database) - logInfo(boldwhite("Corediff loaded ", len(db), " precomputed hashes. (C) 2020-2023 labs@sansec.io")) + logInfo(boldwhite("Corediff ", corediffVersion, " loaded ", len(db), " precomputed hashes. (C) 2020-2023 labs@sansec.io")) logInfo("Using database:", args.Database, "\n") if args.Merge { diff --git a/corediff_test.go b/corediff_test.go index 7f4ee91..8af8390 100644 --- a/corediff_test.go +++ b/corediff_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -func digest(b uint32) string { +func digest(b uint64) string { return fmt.Sprintf("%x", b) } @@ -28,7 +28,7 @@ func Test_hash(t *testing.T) { args []byte want string }{ - {[]byte("banaan"), "14ac6691"}, + {[]byte("banaan"), "acfb1ff4438e39f3"}, } for _, tt := range tests { t.Run(string(tt.args), func(t *testing.T) { @@ -42,7 +42,7 @@ func Test_hash(t *testing.T) { func Test_vendor_bug(t *testing.T) { db := loadDB("fixture/sample.db") assert.Len(t, db, 238) - wantHash := uint32(3333369281) + wantHash := uint64(3900178074848893275) if _, ok := db[wantHash]; !ok { t.Error("hash not in db") } diff --git a/fixture/sample.db b/fixture/sample.db index d395b42..b216a96 100644 Binary files a/fixture/sample.db and b/fixture/sample.db differ diff --git a/go.mod b/go.mod index f08a892..17ed5e8 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,26 @@ -module github.com/gwillem/magento-corediff +module github.com/sansecio/corediff go 1.20 require ( - github.com/cespare/xxhash/v2 v2.2.0 - github.com/fatih/color v1.12.0 + github.com/fatih/color v1.15.0 github.com/gobwas/glob v0.2.3 - github.com/gwillem/urlfilecache v0.0.0-20201121123616-8e4f7e58333d + github.com/gwillem/go-buildversion v0.0.0-20230402114832-b1addefa8764 + github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292 + github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13 github.com/jessevdk/go-flags v1.5.0 - github.com/stretchr/testify v1.6.1 + github.com/stretchr/testify v1.8.2 ) require ( - github.com/adrg/xdg v0.2.3 // indirect - github.com/davecgh/go-spew v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect + github.com/adrg/xdg v0.4.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + golang.org/x/sys v0.6.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) + +//replace github.com/gwillem/go-buildversion => ../go-buildversion diff --git a/go.sum b/go.sum index e96fc2f..dd25a70 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,47 @@ -github.com/adrg/xdg v0.2.3 h1:GxXngdYxNDkoUvZXjNJGwqZxWXi43MKbOOlA/00qZi4= -github.com/adrg/xdg v0.2.3/go.mod h1:7I2hH/IT30IsupOpKZ5ue7/qNi3CoKzD6tL3HwpaRMQ= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/gwillem/urlfilecache v0.0.0-20201121123616-8e4f7e58333d h1:SUYp/bOtCvpddj20LJRLoU/YFnoQhXLGG3S+Ro8uWP0= -github.com/gwillem/urlfilecache v0.0.0-20201121123616-8e4f7e58333d/go.mod h1:wcspENTUwvMi317k0q22DWfx6hIXAlEjz5LO6pjJWQ0= +github.com/gwillem/go-buildversion v0.0.0-20230402113215-08d191152cc5 h1:tMpRA020WZ+HsqrRFiflnC0Pj9gseMrG8jWimXw9WWQ= +github.com/gwillem/go-buildversion v0.0.0-20230402113215-08d191152cc5/go.mod h1:mT3drjAmZ/WgzCwPqf270xC9absLnjSMbDSPXPRiGk8= +github.com/gwillem/go-buildversion v0.0.0-20230402114832-b1addefa8764 h1:GL/gS9rv5F1oBgQwbT+pl0HefNbDZLjbcWmDNT/pUOU= +github.com/gwillem/go-buildversion v0.0.0-20230402114832-b1addefa8764/go.mod h1:yf2A6rXn1ptErQTutA0vpEDZJG6E0b/aUnMnXelY5tU= +github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292 h1:AoF/jIYBBxpLJkyf204k6ArAFDFVKitkcOGnyYuWHco= +github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292/go.mod h1:Zv9vnDD/HDN/WCidxG2QdCEDtYIaIeUTX9yHspWZfEM= +github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13 h1:Kuu6BZRS1uaSatweU/S5pWKkwt+QFCMb2QjJrWcvTWA= +github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13/go.mod h1:vEuNDp+GXXq7N5Anwiu1f4VQ08IGuSTGs/PaFCEno/U= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/helpers.go b/helpers.go index 99a91d6..192a91f 100644 --- a/helpers.go +++ b/helpers.go @@ -3,12 +3,12 @@ package main import ( "bytes" "fmt" - "hash/crc32" "os" "path/filepath" "regexp" "strings" + "github.com/cespare/xxhash/v2" "github.com/gobwas/glob" ) @@ -78,11 +78,11 @@ func check(err error) { } } -func hash(b []byte) uint32 { - return crc32.ChecksumIEEE(b) +func hash(b []byte) uint64 { + return xxhash.Sum64(b) } -func pathHash(p string) uint32 { +func pathHash(p string) uint64 { return hash([]byte("path:" + p)) } diff --git a/needle_test.go b/needle_test.go new file mode 100644 index 0000000..09006ba --- /dev/null +++ b/needle_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + "testing" +) + +func Test_Needle(t *testing.T) { + + needles := []string{ + "path:app/code/Magedelight/GeoIp/Controller/Adminhtml/Currencymapping/Delete.php", + } + dbpath := "m2.db" + + db := loadDB(dbpath) + fmt.Println("Loaded entries:", len(db)) + + for k, _ := range db { + fmt.Println("first entry", k) + break + } + + for _, needle := range needles { + checksum := hash([]byte(needle)) + hash := fmt.Sprintf("%x", checksum) + _, ok := db[checksum] + fmt.Println(ok, hash) + } +} diff --git a/setup.go b/setup.go index 3ede7d9..8e7995f 100644 --- a/setup.go +++ b/setup.go @@ -11,7 +11,7 @@ import ( ) type ( - hashDB map[uint32]struct{} + hashDB map[uint64]struct{} walkStats struct { totalFiles int @@ -121,7 +121,7 @@ func setup() *baseArgs { fmt.Println("Can't merge without given --database file") os.Exit(1) } - fmt.Println("Downloading default hash database from", hashDBURL) + // fmt.Println("Using default hash database from", hashDBURL) args.Database = urlfilecache.ToPath(hashDBURL) }