diff --git a/README.md b/README.md index 9ff2ed54..45193163 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Usage: go list -m all | nancy [options] go list -m all | nancy iq [options] + nancy config nancy [options] nancy [options] @@ -36,6 +37,10 @@ Options: Styling for output format. ["json" "json-pretty" "text" "csv"] (default "text") -quiet indicate output should contain only packages with vulnerabilities + -token string + Specify OSS Index API token for request + -user string + Specify OSS Index username for request -v Set log level to Info -version prints current nancy version @@ -58,9 +63,9 @@ Options: -stage string Specify stage for application (default "develop") -token string - Specify token/password for request (default "admin123") + Specify Nexus IQ token/password for request (default "admin123") -user string - Specify username for request (default "admin") + Specify Nexus IQ username for request (default "admin") -v Set log level to Info -vv Set log level to Debug @@ -100,6 +105,30 @@ We publish a few different flavors for convenience: ### OSS Index Options +#### Rate limiting / Setting OSS Index config + +**NOTE: New as of Nancy v0.1.17** + +If you start using Nancy extensively, you might run into Rate Limiting from OSS Index! Don't worry, we've got your back! + +If you run into Rate Limiting you should recieve an error that will give you instructions on how to register on OSS Index: + +``` +You have been rate limited by OSS Index. +If you do not have a OSS Index account, please visit https://ossindex.sonatype.org/user/register to register an account. +After registering and verifying your account, you can retrieve your username (Email Address), and API Token +at https://ossindex.sonatype.org/user/settings. Upon retrieving those, run 'nancy config', set your OSS Index +settings, and rerun Nancy. +``` + +After setting this config, you'll be gifted a nice new higher rate limit. If you escape this limit, you might take a look at using Nexus IQ Server, or reach out to the friendly people at OSS Index for partnership opportunities. + +You can also set the user and token via the command line like so: + +`nancy -user auser@anemailaddress.com -token A4@k3@p1T0k3n` + +This can be handy for testing your account out, or if you want to override your set config with a different user. + #### Quiet mode You can run `nancy` in a quiet manner, only getting back a list of vulnerable components by running: @@ -144,46 +173,6 @@ CVN-111 until=2021-01-01 CVN-543 until=2018-02-12 #Waiting on release from third party. Should be out before this date but gives us a little time to fix it. ``` -### Nexus IQ Server Options - -By default, assuming you have an out of the box Nexus IQ Server running, you can run `nancy` like so: - -`go list -m all | ./nancy iq -application public-application-id` - -It is STRONGLY suggested that you do not do this, and we will warn you on output if you are. - -A more logical use of `nancy` against Nexus IQ Server will look like so: - -`go list -m all | ./nancy iq -application public-application-id -user nondefaultuser -token yourtoken -server-url http://adifferentserverurl:port -stage develop` - -Options for stage are as follows: - -`build, develop, stage-release, release` - -By default `-stage` will be `develop`. - -Successful submissions to Nexus IQ Server will result in either an OS exit of 0, meaning all is clear and a response akin to: - -``` -Wonderbar! No policy violations reported for this audit! -Report URL: http://reportURL -``` - -Failed submissions will either indicate failure because of an issue with processing the request, or a policy violation. Both will exit with a code of 1, allowing you to fail your build in CI. Policy Violation failures will include a report URL where you can learn more about why you encountered a failure. - -Policy violations will look like: - -``` -Hi, Nancy here, you have some policy violations to clean up! -Report URL: http://reportURL -``` - -Errors processing in Nexus IQ Server will look like: - -``` -Uh oh! There was an error with your request to Nexus IQ Server: -``` - #### Output We support multiple different output formats. Examples can be found below for each. [This intentionally vulnerable repo](https://github.com/sonatype-nexus-community/intentionally-vulnerable-golang-project) was used to generate the example output. @@ -353,7 +342,55 @@ Count,Package,Is Vulnerable,Num Vulnerabilities,Vulnerabilities [3/10],pkg:golang/github.com/ethereum/go-ethereum@1.8.15,true,1,"[{""Id"":""4efaed86-e62e-4c0c-b812-36c07e61ede4"",""Title"":""CWE-400: Uncontrolled Resource Consumption ('Resource Exhaustion')"",""Description"":""The software does not properly restrict the size or amount of resources that are requested or influenced by an actor, which can be used to consume more resources than intended."",""CvssScore"":""7.5"",""CvssVector"":""CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"",""Cve"":"""",""Reference"":""https://ossindex.sonatype.org/vuln/4efaed86-e62e-4c0c-b812-36c07e61ede4"",""Excluded"":false}]" ... ``` +### Nexus IQ Server Options + +By default, assuming you have an out of the box Nexus IQ Server running, you can run `nancy` like so: + +`go list -m all | ./nancy iq -application public-application-id` + +It is STRONGLY suggested that you do not do this, and we will warn you on output if you are. + +A more logical use of `nancy` against Nexus IQ Server will look like so: + +`go list -m all | ./nancy iq -application public-application-id -user nondefaultuser -token yourtoken -server-url http://adifferentserverurl:port -stage develop` + +Options for stage are as follows: + +`build, develop, stage-release, release` + +By default `-stage` will be `develop`. + +Successful submissions to Nexus IQ Server will result in either an OS exit of 0, meaning all is clear and a response akin to: + +``` +Wonderbar! No policy violations reported for this audit! +Report URL: http://reportURL +``` + +Failed submissions will either indicate failure because of an issue with processing the request, or a policy violation. Both will exit with a code of 1, allowing you to fail your build in CI. Policy Violation failures will include a report URL where you can learn more about why you encountered a failure. + +Policy violations will look like: + +``` +Hi, Nancy here, you have some policy violations to clean up! +Report URL: http://reportURL +``` + +Errors processing in Nexus IQ Server will look like: + +``` +Uh oh! There was an error with your request to Nexus IQ Server: +``` + +#### Persistent Nexus IQ Server Config + +Nancy let's you set the Nexus IQ Server Address, User and Token as persistent config (application and stage are generally per project so we do not let you set these globally). + +To set your Nexus IQ Server config run: + +`nancy config` +Choose `iq` as an option and run through the rest of the config. Once you are done, Nancy should use this config for communicating with Nexus IQ, simplifying your use of the tool. ### Usage in CI diff --git a/configuration/parse.go b/configuration/parse.go index 2878b602..6d0368c0 100644 --- a/configuration/parse.go +++ b/configuration/parse.go @@ -18,7 +18,9 @@ import ( "errors" "flag" "fmt" + "io/ioutil" "os" + "path/filepath" "reflect" "regexp" "strings" @@ -26,7 +28,9 @@ import ( "github.com/sirupsen/logrus" "github.com/sonatype-nexus-community/nancy/audit" + . "github.com/sonatype-nexus-community/nancy/logger" "github.com/sonatype-nexus-community/nancy/types" + "gopkg.in/yaml.v2" ) type Configuration struct { @@ -42,6 +46,8 @@ type Configuration struct { Info bool Debug bool Trace bool + Username string `yaml:"Username"` + Token string `yaml:"Token"` } type IqConfiguration struct { @@ -66,8 +72,8 @@ func ParseIQ(args []string) (config IqConfiguration, err error) { iqCommand.BoolVar(&config.Info, "v", false, "Set log level to Info") iqCommand.BoolVar(&config.Debug, "vv", false, "Set log level to Debug") iqCommand.BoolVar(&config.Trace, "vvv", false, "Set log level to Trace") - iqCommand.StringVar(&config.User, "user", "admin", "Specify username for request") - iqCommand.StringVar(&config.Token, "token", "admin123", "Specify token/password for request") + iqCommand.StringVar(&config.User, "user", "admin", "Specify Nexus IQ username for request") + iqCommand.StringVar(&config.Token, "token", "admin123", "Specify Nexus IQ token/password for request") iqCommand.StringVar(&config.Server, "server-url", "http://localhost:8070", "Specify Nexus IQ Server URL/port") iqCommand.StringVar(&config.Application, "application", "", "Specify application ID for request") iqCommand.StringVar(&config.Stage, "stage", "develop", "Specify stage for application") @@ -83,6 +89,14 @@ Options: os.Exit(2) } + ConfigLocation = filepath.Join(HomeDir, types.IQServerDirName, types.IQServerConfigFileName) + + err = loadIQConfigFromFile(ConfigLocation, &config) + if err != nil { + fmt.Println(err) + LogLady.Info("Unable to load config from file") + } + err = iqCommand.Parse(args) if err != nil { return config, err @@ -91,6 +105,32 @@ Options: return config, nil } +func loadConfigFromFile(configLocation string, config *Configuration) error { + b, err := ioutil.ReadFile(configLocation) + if err != nil { + return err + } + err = yaml.Unmarshal(b, config) + if err != nil { + return err + } + + return nil +} + +func loadIQConfigFromFile(configLocation string, config *IqConfiguration) error { + b, err := ioutil.ReadFile(configLocation) + if err != nil { + return err + } + err = yaml.Unmarshal(b, config) + if err != nil { + return err + } + + return nil +} + func Parse(args []string) (Configuration, error) { config := Configuration{} var excludeVulnerabilityFilePath string @@ -112,6 +152,8 @@ func Parse(args []string) (Configuration, error) { flag.BoolVar(&config.Debug, "vv", false, "Set log level to Debug") flag.BoolVar(&config.Trace, "vvv", false, "Set log level to Trace") flag.Var(&config.CveList, "exclude-vulnerability", "Comma separated list of CVEs to exclude") + flag.StringVar(&config.Username, "user", "", "Specify OSS Index username for request") + flag.StringVar(&config.Token, "token", "", "Specify OSS Index API token for request") flag.StringVar(&excludeVulnerabilityFilePath, "exclude-vulnerability-file", "./.nancy-ignore", "Path to a file containing newline separated CVEs to be excluded") flag.StringVar(&outputFormat, "output", "text", "Styling for output format. "+fmt.Sprintf("%+q", reflect.ValueOf(outputFormats).MapKeys())) @@ -119,6 +161,7 @@ func Parse(args []string) (Configuration, error) { _, _ = fmt.Fprintf(os.Stderr, `Usage: go list -m all | nancy [options] go list -m all | nancy iq [options] + nancy config nancy [options] nancy [options] @@ -128,7 +171,15 @@ Options: os.Exit(2) } - err := flag.CommandLine.Parse(args) + ConfigLocation = filepath.Join(HomeDir, types.OssIndexDirName, types.OssIndexConfigFileName) + + err := loadConfigFromFile(ConfigLocation, &config) + if err != nil { + fmt.Println(err) + LogLady.Info("Unable to load config from file") + } + + err = flag.CommandLine.Parse(args) if err != nil { return config, err } diff --git a/configuration/parse_test.go b/configuration/parse_test.go index 9d8f2aa1..ede34ca0 100644 --- a/configuration/parse_test.go +++ b/configuration/parse_test.go @@ -140,5 +140,7 @@ func TestConfigParseIQ(t *testing.T) { } func setup() { + // Set HomeDir to a nonsensical location to avoid loading file based config + HomeDir = "/doesnt/exist" flag.CommandLine = flag.NewFlagSet("", flag.ContinueOnError) } diff --git a/configuration/set.go b/configuration/set.go new file mode 100644 index 00000000..abe27f34 --- /dev/null +++ b/configuration/set.go @@ -0,0 +1,175 @@ +package configuration + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + . "github.com/sonatype-nexus-community/nancy/logger" + "github.com/sonatype-nexus-community/nancy/types" + "gopkg.in/yaml.v2" +) + +// IQConfig is a struct for holding IQ Configuration, and for writing it to yaml +type IQConfig struct { + Server string `yaml:"Server"` + Username string `yaml:"Username"` + Token string `yaml:"Token"` +} + +// OSSIndexConfig is a struct for holding OSS Index Configuration, and for writing it to yaml +type OSSIndexConfig struct { + Username string `yaml:"Username"` + Token string `yaml:"Token"` +} + +var ( + // HomeDir is exported so that in testing it can be set to a location like /tmp + HomeDir string + // ConfigLocation is exported so that in testing it can be used to test if the file has been written properly + ConfigLocation string +) + +func init() { + HomeDir, _ = os.UserHomeDir() +} + +// GetConfigFromCommandLine is a method to obtain IQ or OSS Index config from the command line, +// and then write it to disk. +func GetConfigFromCommandLine(stdin io.Reader) (err error) { + LogLady.Info("Starting process to obtain config from user") + reader := bufio.NewReader(stdin) + fmt.Print("Hi! What config can I help you set, IQ or OSS Index (values: iq, ossindex, enter for exit)? ") + configType, _ := reader.ReadString('\n') + + switch str := strings.TrimSpace(configType); str { + case "iq": + LogLady.Info("User chose to set IQ Config, moving forward") + ConfigLocation = filepath.Join(HomeDir, types.IQServerDirName, types.IQServerConfigFileName) + err = getAndSetIQConfig(reader) + case "ossindex": + LogLady.Info("User chose to set OSS Index config, moving forward") + ConfigLocation = filepath.Join(HomeDir, types.OssIndexDirName, types.OssIndexConfigFileName) + err = getAndSetOSSIndexConfig(reader) + case "": + return + default: + LogLady.Info("User chose to set OSS Index config, moving forward") + fmt.Println("Invalid value, 'iq' and 'ossindex' are accepted values, try again!") + GetConfigFromCommandLine(stdin) + } + + if err != nil { + LogLady.Error(err) + return + } + return +} + +func getAndSetIQConfig(reader *bufio.Reader) (err error) { + LogLady.Info("Getting config for IQ Server from user") + + iqConfig := IQConfig{Server: "http://localhost:8070", Username: "admin", Token: "admin123"} + + fmt.Print("What is the address of your Nexus IQ Server (default: http://localhost:8070)? ") + server, _ := reader.ReadString('\n') + iqConfig.Server = emptyOrDefault(server, iqConfig.Server) + + fmt.Print("What username do you want to authenticate as (default: admin)? ") + username, _ := reader.ReadString('\n') + iqConfig.Username = emptyOrDefault(username, iqConfig.Username) + + fmt.Print("What token do you want to use (default: admin123)? ") + token, _ := reader.ReadString('\n') + iqConfig.Token = emptyOrDefault(token, iqConfig.Token) + + if iqConfig.Username == "admin" || iqConfig.Token == "admin123" { + LogLady.Info("Warning user of bad life choices, using default values for IQ Server username or token") + warnUserOfBadLifeChoices() + fmt.Print("[y/N]? ") + theChoice, _ := reader.ReadString('\n') + theChoice = emptyOrDefault(theChoice, "y") + if theChoice == "y" { + LogLady.Info("User chose to rectify their bad life choices, asking for config again") + getAndSetIQConfig(reader) + } else { + LogLady.Info("Successfully got IQ Server config from user, attempting to save to disk") + err = marshallAndWriteToDisk(iqConfig) + } + } else { + LogLady.Info("Successfully got IQ Server config from user, attempting to save to disk") + err = marshallAndWriteToDisk(iqConfig) + } + + if err != nil { + return + } + return +} + +func emptyOrDefault(value string, defaultValue string) string { + str := strings.Trim(strings.TrimSpace(value), "\n") + if str == "" { + return defaultValue + } + return str +} + +func getAndSetOSSIndexConfig(reader *bufio.Reader) (err error) { + LogLady.Info("Getting config for OSS Index from user") + + ossIndexConfig := OSSIndexConfig{} + + fmt.Print("What username do you want to authenticate as (ex: admin)? ") + ossIndexConfig.Username, _ = reader.ReadString('\n') + ossIndexConfig.Username = strings.Trim(strings.TrimSpace(ossIndexConfig.Username), "\n") + + fmt.Print("What token do you want to use? ") + ossIndexConfig.Token, _ = reader.ReadString('\n') + ossIndexConfig.Token = strings.Trim(strings.TrimSpace(ossIndexConfig.Token), "\n") + + LogLady.Info("Successfully got OSS Index config from user, attempting to save to disk") + err = marshallAndWriteToDisk(ossIndexConfig) + if err != nil { + LogLady.Error(err) + return + } + + return +} + +func marshallAndWriteToDisk(config interface{}) (err error) { + d, err := yaml.Marshal(config) + if err != nil { + return err + } + + base := filepath.Dir(ConfigLocation) + + if _, err := os.Stat(base); os.IsNotExist(err) { + os.Mkdir(base, os.ModePerm) + } + + err = ioutil.WriteFile(ConfigLocation, d, 0644) + if err != nil { + return + } + + LogLady.WithField("config_location", ConfigLocation).Info("Successfully wrote config to disk") + fmt.Println(fmt.Sprintf("Successfully wrote config to: %s", ConfigLocation)) + return +} + +func warnUserOfBadLifeChoices() { + fmt.Println() + fmt.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + fmt.Println("!!!! WARNING : You are using the default username and/or password for Nexus IQ. !!!!") + fmt.Println("!!!! You are strongly encouraged to change these, and use a token. !!!!") + fmt.Println("!!!! Would you like to change them and try again? !!!!") + fmt.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + fmt.Println() +} diff --git a/configuration/set_test.go b/configuration/set_test.go new file mode 100644 index 00000000..d615a1da --- /dev/null +++ b/configuration/set_test.go @@ -0,0 +1,87 @@ +package configuration + +import ( + "bytes" + "io/ioutil" + "testing" + + "gopkg.in/yaml.v2" +) + +func TestGetConfigFromCommandLineOssIndex(t *testing.T) { + HomeDir = "/tmp" + var buffer bytes.Buffer + buffer.Write([]byte("ossindex\ntestuser\ntoken\n")) + + err := GetConfigFromCommandLine(&buffer) + if err != nil { + t.Errorf("Test failed: %s", err.Error()) + } + + var ossIndexConfig OSSIndexConfig + + b, err := ioutil.ReadFile(ConfigLocation) + if err != nil { + t.Errorf("Test failed: %s", err.Error()) + } + err = yaml.Unmarshal(b, &ossIndexConfig) + if err != nil { + t.Errorf("Test failed: %s", err.Error()) + } + + if ossIndexConfig.Username != "testuser" && ossIndexConfig.Token != "token" { + t.Errorf("Config not set properly, expected 'testuser' && 'token' but got %s and %s", ossIndexConfig.Username, ossIndexConfig.Token) + } +} + +func TestGetConfigFromCommandLineIqServer(t *testing.T) { + HomeDir = "/tmp" + var buffer bytes.Buffer + buffer.Write([]byte("iq\nhttp://localhost:8070\nadmin\nadmin123\nn")) + + err := GetConfigFromCommandLine(&buffer) + if err != nil { + t.Errorf("Test failed: %s", err.Error()) + } + + var iqConfig IQConfig + + b, err := ioutil.ReadFile(ConfigLocation) + if err != nil { + t.Errorf("Test failed: %s", err.Error()) + } + err = yaml.Unmarshal(b, &iqConfig) + if err != nil { + t.Errorf("Test failed: %s", err.Error()) + } + + if iqConfig.Username != "admin" && iqConfig.Token != "admin123" && iqConfig.Server != "http://localhost:8070" { + t.Errorf("Config not set properly, expected 'admin', 'admin123' and 'http://localhost:8070' but got %s, %s and %s", iqConfig.Username, iqConfig.Token, iqConfig.Server) + } +} + +func TestGetConfigFromCommandLineIqServerWithLoopToResetConfig(t *testing.T) { + HomeDir = "/tmp" + var buffer bytes.Buffer + buffer.Write([]byte("iq\nhttp://localhost:8070\nadmin\nadmin123\ny\nhttp://localhost:8080\nadmin1\nadmin1234\n")) + + err := GetConfigFromCommandLine(&buffer) + if err != nil { + t.Errorf("Test failed: %s", err.Error()) + } + + var iqConfig IQConfig + + b, err := ioutil.ReadFile(ConfigLocation) + if err != nil { + t.Errorf("Test failed: %s", err.Error()) + } + err = yaml.Unmarshal(b, &iqConfig) + if err != nil { + t.Errorf("Test failed: %s", err.Error()) + } + + if iqConfig.Username != "admin1" && iqConfig.Token != "admin1234" && iqConfig.Server != "http://localhost:8080" { + t.Errorf("Config not set properly, expected 'admin1', 'admin1234' and 'http://localhost:8080' but got %s, %s and %s", iqConfig.Username, iqConfig.Token, iqConfig.Server) + } +} diff --git a/go.mod b/go.mod index da1e48fb..9ba91e27 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/google/go-cmp v0.3.1 // indirect github.com/jarcoal/httpmock v1.0.4 github.com/jmank88/nuts v0.3.0 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/logrusorgru/aurora v0.0.0-20190803045625-94edacc10f9b github.com/nightlyone/lockfile v0.0.0-20180618180623-0ad87eef1443 // indirect github.com/package-url/packageurl-go v0.1.0 @@ -24,13 +25,14 @@ require ( github.com/pkg/errors v0.8.0 // indirect github.com/sdboyer/constext v0.0.0-20170321163424-836a14457353 // indirect github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 - github.com/sirupsen/logrus v1.4.2 + github.com/sirupsen/logrus v1.5.0 github.com/spf13/afero v1.2.2 // indirect github.com/stretchr/testify v1.3.0 golang.org/x/net v0.0.0-20181220203305-927f97764cc3 // indirect golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect - golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect + golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa // indirect gopkg.in/go-playground/assert.v1 v1.2.1 + gopkg.in/yaml.v2 v2.2.8 ) go 1.13 diff --git a/go.sum b/go.sum index 2d9cb311..d119f507 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/jmank88/nuts v0.3.0 h1:UZUboV1LXVkBUTHLRTEZrDfAL7QYgj9jEsBCiJHrxEM= github.com/jmank88/nuts v0.3.0/go.mod h1:kTf5cyoLibZUQg9Lns/gteKO1d/5XrhacD1QVKviAKk= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/logrusorgru/aurora v0.0.0-20190803045625-94edacc10f9b h1:PMbSa9CgaiQR9NLlUTwKi+7aeLl3GG5JX5ERJxfQ3IE= github.com/logrusorgru/aurora v0.0.0-20190803045625-94edacc10f9b/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/nightlyone/lockfile v0.0.0-20180618180623-0ad87eef1443 h1:+2OJrU8cmOstEoh0uQvYemRGVH1O6xtO2oANUWHFnP0= @@ -53,12 +55,11 @@ github.com/sdboyer/constext v0.0.0-20170321163424-836a14457353 h1:tnWWLf0nI2TI62 github.com/sdboyer/constext v0.0.0-20170321163424-836a14457353/go.mod h1:5HStXbIikwtDAgAIqiQIqVgMn7mlvZa6PTpwiAVYGYg= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.5.0 h1:1N5EYkVAPEywqZRJd7cwnRtCb6xJx7NH3T3WUTF980Q= +github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -67,8 +68,8 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa h1:mQTN3ECqfsViCNBgq+A40vdwhkGykrrQlYe3mPj6BoU= +golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= @@ -77,3 +78,5 @@ gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXa gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/logger/logger.go b/logger/logger.go index 78aab8ba..9ff85545 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -17,11 +17,12 @@ package logger import ( "fmt" - "github.com/sirupsen/logrus" - "github.com/sonatype-nexus-community/nancy/types" "os" "path" "strings" + + "github.com/sirupsen/logrus" + "github.com/sonatype-nexus-community/nancy/types" ) const DefaultLogFilename = "nancy.combined.log" @@ -42,7 +43,8 @@ func doInit(args []string) { if useTestLogFile(args) { DefaultLogFile = TestLogfilename } - file, err := os.OpenFile(GetLogFileLocation(), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666) + + file, err := os.OpenFile(GetLogFileLocation(), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666) if err != nil { fmt.Print(err) } diff --git a/logger/logger_test.go b/logger/logger_test.go index 7cad3014..10b75159 100644 --- a/logger/logger_test.go +++ b/logger/logger_test.go @@ -17,27 +17,20 @@ package logger import ( "encoding/json" - "github.com/stretchr/testify/assert" "io/ioutil" "os" "strings" "testing" - "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" ) func TestLogger(t *testing.T) { - LogLady.Level = logrus.DebugLevel - // Do initial write to have a file - LogLady.Debug("Test") - if !strings.Contains(GetLogFileLocation(), TestLogfilename) { t.Errorf("Nancy test file not in log file location. args: %+v", os.Args) } - err := os.Truncate(GetLogFileLocation(), 0) - - LogLady.Debug("Test") + LogLady.Info("Test") dat, err := ioutil.ReadFile(GetLogFileLocation()) if err != nil { @@ -51,7 +44,7 @@ func TestLogger(t *testing.T) { t.Error("Improperly written log, should be valid json") } - if logTest.Level != "debug" { + if logTest.Level != "info" { t.Error("Log level not set properly") } diff --git a/main.go b/main.go index 0068afb0..d669f54d 100644 --- a/main.go +++ b/main.go @@ -52,6 +52,12 @@ func main() { LogLady.WithField("config", config).Info("Obtained IQ config") processIQConfig(config) LogLady.Info("Nancy finished parsing config for IQ") + } else if len(os.Args) > 1 && os.Args[1] == "config" { + LogLady.Info("Nancy setting config via the command line") + err := configuration.GetConfigFromCommandLine(os.Stdin) + customerrors.Check(err, "Unable to set config for Nancy") + + os.Exit(0) } else { LogLady.Info("Nancy parsing config for OSS Index") ossIndexConfig, err := configuration.Parse(os.Args[1:]) @@ -268,7 +274,7 @@ func doCheckExistenceAndParse(config configuration.Configuration) { func checkOSSIndex(purls []string, invalidpurls []string, config configuration.Configuration) { var packageCount = len(purls) - coordinates, err := ossindex.AuditPackages(purls) + coordinates, err := ossindex.AuditPackagesWithOSSIndex(purls, &config) customerrors.Check(err, "Error auditing packages") var invalidCoordinates []types.Coordinate diff --git a/ossindex/ossindex.go b/ossindex/ossindex.go index 8989b9c4..fbd72992 100644 --- a/ossindex/ossindex.go +++ b/ossindex/ossindex.go @@ -29,6 +29,7 @@ import ( "time" "github.com/dgraph-io/badger" + "github.com/sonatype-nexus-community/nancy/configuration" "github.com/sonatype-nexus-community/nancy/customerrors" . "github.com/sonatype-nexus-community/nancy/logger" "github.com/sonatype-nexus-community/nancy/types" @@ -86,8 +87,20 @@ func openDb(dbDir string) (db *badger.DB, err error) { return } -// AuditPackages will given a list of Package URLs, run an OSS Index audit +// AuditPackages will given a list of Package URLs, run an OSS Index audit. +// +// Deprecated: AuditPackages is old and being maintained for upstream compatibility at the moment. +// It will be removed when we go to a major version release. Use AuditPackagesWithOSSIndex instead. func AuditPackages(purls []string) ([]types.Coordinate, error) { + return doAuditPackages(purls, nil) +} + +// AuditPackagesWithOSSIndex will given a list of Package URLs, run an OSS Index audit, and takes OSS Index configuration +func AuditPackagesWithOSSIndex(purls []string, config *configuration.Configuration) ([]types.Coordinate, error) { + return doAuditPackages(purls, config) +} + +func doAuditPackages(purls []string, config *configuration.Configuration) ([]types.Coordinate, error) { dbDir := getDatabaseDirectory() if err := os.MkdirAll(dbDir, os.ModePerm); err != nil { return nil, err @@ -133,55 +146,23 @@ func AuditPackages(purls []string) ([]types.Coordinate, error) { LogLady.WithField("request", request).Info("Prepping request to OSS Index") var jsonStr, _ = json.Marshal(request) - req, err := setupRequest(jsonStr) + coordinates, err := doRequestToOSSIndex(jsonStr, config) if err != nil { return nil, err } - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - LogLady.WithField("resp_status_code", resp.Status).Error("Error accessing OSS Index") - return nil, fmt.Errorf("[%s] error accessing OSS Index", resp.Status) - } - - LogLady.WithField("status_code", resp.StatusCode).Info("Obtained a response from OSS Index") - - defer func() { - if err := resp.Body.Close(); err != nil { - LogLady.WithField("error", err).Error("Error closing response body") - } - }() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - LogLady.WithField("error", err).Error("Error accessing OSS Index") - return nil, err - } - - // Process results - var coordinates []types.Coordinate - if err = json.Unmarshal([]byte(body), &coordinates); err != nil { - LogLady.WithField("error", err).Error("Unable to unmarshall body into coordinates") - return nil, err - } - LogLady.WithField("coordinates", coordinates).Info("Coordinates unmarshalled from OSS Index") // Cache the new results if err := db.Update(func(txn *badger.Txn) error { for i := 0; i < len(coordinates); i++ { - var coord = coordinates[i].Coordinates + coord := coordinates[i].Coordinates results = append(results, coordinates[i]) - var coordJson, _ = json.Marshal(coordinates[i]) - LogLady.WithField("json", coordJson).Info("Marshall coordinate into json for insertion into DB") + coordJSON, _ := json.Marshal(coordinates[i]) + LogLady.WithField("json", coordinates[i]).Info("Marshall coordinate into json for insertion into DB") - err := txn.SetWithTTL([]byte(strings.ToLower(coord)), []byte(coordJson), time.Hour*12) + err := txn.SetWithTTL([]byte(strings.ToLower(coord)), []byte(coordJSON), time.Hour*12) if err != nil { LogLady.WithField("error", err).Error("Unable to add coordinate to cache DB") return err @@ -197,23 +178,49 @@ func AuditPackages(purls []string) ([]types.Coordinate, error) { return results, nil } -func chunk(purls []string, chunkSize int) [][]string { - var divided [][]string +func doRequestToOSSIndex(jsonStr []byte, config *configuration.Configuration) (coordinates []types.Coordinate, err error) { + req, err := setupRequest(jsonStr, config) + if err != nil { + return + } - for i := 0; i < len(purls); i += chunkSize { - end := i + chunkSize + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return + } - if end > len(purls) { - end = len(purls) + if resp.StatusCode == http.StatusTooManyRequests { + LogLady.WithField("resp_status_code", resp.Status).Error("Error accessing OSS Index due to Rate Limiting") + return nil, &types.OSSIndexRateLimitError{} + } + + if resp.StatusCode != http.StatusOK { + LogLady.WithField("resp_status_code", resp.Status).Error("Error accessing OSS Index") + return nil, fmt.Errorf("[%s] error accessing OSS Index", resp.Status) + } + + defer func() { + if err := resp.Body.Close(); err != nil { + LogLady.WithField("error", err).Error("Error closing response body") } + }() - divided = append(divided, purls[i:end]) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + LogLady.WithField("error", err).Error("Error accessing OSS Index") + return } - return divided + // Process results + if err = json.Unmarshal([]byte(body), &coordinates); err != nil { + LogLady.WithField("error", err).Error("Error unmarshalling response from OSS Index") + return + } + return } -func setupRequest(jsonStr []byte) (req *http.Request, err error) { +func setupRequest(jsonStr []byte, config *configuration.Configuration) (req *http.Request, err error) { LogLady.WithField("json_string", string(jsonStr)).Debug("Setting up new POST request to OSS Index") req, err = http.NewRequest( "POST", @@ -226,6 +233,26 @@ func setupRequest(jsonStr []byte) (req *http.Request, err error) { req.Header.Set("User-Agent", useragent.GetUserAgent()) req.Header.Set("Content-Type", "application/json") + if config != nil && config.Username != "" && config.Token != "" { + LogLady.Info("Set OSS Index Basic Auth") + req.SetBasicAuth(config.Username, config.Token) + } return req, nil } + +func chunk(purls []string, chunkSize int) [][]string { + var divided [][]string + + for i := 0; i < len(purls); i += chunkSize { + end := i + chunkSize + + if end > len(purls) { + end = len(purls) + } + + divided = append(divided, purls[i:end]) + } + + return divided +} diff --git a/ossindex/ossindex_test.go b/ossindex/ossindex_test.go index 8191fffa..4cabe1c1 100644 --- a/ossindex/ossindex_test.go +++ b/ossindex/ossindex_test.go @@ -26,6 +26,7 @@ import ( "time" "github.com/dgraph-io/badger" + "github.com/sonatype-nexus-community/nancy/configuration" "github.com/sonatype-nexus-community/nancy/types" "github.com/stretchr/testify/assert" ) @@ -251,10 +252,15 @@ func TestAuditPackages_SinglePackage_Cached_WithExpiredTTL(t *testing.T) { func TestSetupRequest(t *testing.T) { coordJson, _ := setupJson(t) - req, err := setupRequest(coordJson) + config := configuration.Configuration{Username: "testuser", Token: "test"} + req, err := setupRequest(coordJson, &config) assert.Equal(t, req.Header.Get("Content-Type"), "application/json") assert.Equal(t, req.Method, "POST") + user, token, ok := req.BasicAuth() + assert.Equal(t, user, "testuser") + assert.Equal(t, token, "test") + assert.Equal(t, ok, true) assert.Nil(t, err) } diff --git a/types/types.go b/types/types.go index b8d193bd..17e14147 100644 --- a/types/types.go +++ b/types/types.go @@ -23,7 +23,10 @@ import ( // Helpful constants to pull strings we use more than once out of code const ( - OssIndexDirName = ".ossindex" + OssIndexDirName = ".ossindex" + OssIndexConfigFileName = ".oss-index-config" + IQServerDirName = ".iqserver" + IQServerConfigFileName = ".iq-server-config" ) type Vulnerability struct { @@ -167,3 +170,16 @@ type Source struct { Name string `xml:"name,attr"` URL string `xml:"v:url"` } + +// OSSIndexRateLimitError is a custom error implementation to allow us to return a better error response to the user +// as well as check the type of the error so we can surface this information. +type OSSIndexRateLimitError struct { +} + +func (o *OSSIndexRateLimitError) Error() string { + return `You have been rate limited by OSS Index. +If you do not have a OSS Index account, please visit https://ossindex.sonatype.org/user/register to register an account. +After registering and verifying your account, you can retrieve your username (Email Address), and API Token +at https://ossindex.sonatype.org/user/settings. Upon retrieving those, run 'nancy config', set your OSS Index +settings, and rerun Nancy.` +}