Adding CMP vendor list loading mechanism & validation against the CMP…

… ID coming inside the consent. (#3)

* Adding CMP vendor list loading mechanism & validation against the CMP ID coming inside the consent.

* Using go install to install the testing dependencies.

* Fixing error message as it's different depending on the environment we run the tests.

.circleci/config.yml

@@ -1,41 +1,44 @@
 version: 2
 jobs:
   unit-testing:
-    working_directory: ~/src/github.com/affectv/iab-tcf
+    working_directory: ~/src/github.com/hybridtheory/iab-tcf
     docker:
-      - image: golang:1.13-stretch
+      - image: golang:1.21-bullseye
         auth:
           username: $DOCKERHUB_USERNAME
           password: $DOCKERHUB_PASSWORD
     steps:
       - checkout
       - restore_cache:
           keys:
-          - v1-go-dependencies-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
+            - v1-go-dependencies-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
       - run:
           name: Install test framework
           command: |
-            go get -u -v github.com/onsi/ginkgo/ginkgo
-            go get -u -v github.com/onsi/gomega
+            go install github.com/onsi/ginkgo/v2/ginkgo
+            go install github.com/onsi/gomega/...
       - run:
           name: Running Unit Tests
           command: |
-            ginkgo -cover -race -outputdir=./ ./...
+            ginkgo --race -cover --junit-report=junit.xml --output-dir=./test-reports ./...
       - run:
           name: Generate code coverage
           command: |
-            echo "mode: set" > coverage.out
-            cat *.coverprofile | grep -v mode: | sort -r | awk '{if($1 != last) {print $0;last=$1}}' >> coverage.out
-            go tool cover -html=coverage.out -o coverage.html
+            go tool cover -html=./test-reports/coverprofile.out -o ./test-reports/coverage.html
       - store_artifacts:
-          path: coverage.html
-          destination: coverage
+          path: ./test-reports/
+          destination: artifacts
       - store_test_results:
           path: test-reports
       - save_cache:
           paths:
             - "/go/pkg/mod"
           key: v1-go-dependencies-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
+      - run:
+          name: codacy coverage
+          command: |
+            export CODACY_PROJECT_NAME=iab-tcf
+            bash <(curl -Lks https://coverage.codacy.com/get.sh) report --force-coverage-parser go -r ./test-reports/coverprofile.out
 
 workflows:
   version: 2

.prettierrc

@@ -0,0 +1,7 @@
+{
+  "printWidth": 120,
+  "tabWidth": 2,
+  "singleQuote": false,
+  "trailingComma": "all",
+  "arrowParens": "always"
+}

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Affectv
+Copyright (c) 2020 Azerion
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,6 +1,7 @@
-# iab-tcf [![CircleCI](https://circleci.com/gh/affectv/iab-tcf.svg?style=svg)](https://circleci.com/gh/affectv/iab-tcf)
+# iab-tcf [![CircleCI](https://circleci.com/gh/hybridtheory/iab-tcf.svg?style=svg)](https://circleci.com/gh/hybridtheory/iab-tcf)
 
-Go code to parse IAB v1/v2 consent based on [github.com/LiveRamp/iabconsent](github.com/LiveRamp/iabconsent) library. 
+Go code to parse IAB v1/v2 consent based on [github.com/LiveRamp/iabconsent](github.com/LiveRamp/iabconsent)
+library.
 
 This package is just a wrapper on their amazing work to fill our needs.
 
@@ -9,7 +10,7 @@ This package is just a wrapper on their amazing work to fill our needs.
 ### Install
 
 ```bash
-go get github.com/affectv/iab-tcf
+go get github.com/hybridtheory/iab-tcf
 ```
 
 ### Example
@@ -19,7 +20,7 @@ package main
 
 import (
     "fmt"
-    iab "github.com/affectv/iab-tcf"
+    iab "github.com/hybridtheory/iab-tcf"
 )
 
 func main() {
@@ -28,9 +29,46 @@ func main() {
 }
 ```
 
+### CMP vendor list
+
+In order to validate if the CMP received belongs to a valid id there's a loader integrated
+with the library.
+
+```golang
+package main
+
+import (
+    "fmt"
+    iab "github.com/hybridtheory/iab-tcf"
+    cmp "github.com/hybridtheory/iab-tcf/cmp"
+)
+
+func main() {
+    cmp.NewLoader().LoadIDs()
+    consent, err := iab.NewConsent(encoded)
+    consent.IsCMPValid() // This will return true/false depending on the ids loaded.
+}
+```
+
+If instead of the default vendor list [https://cmplist.consensu.org/v2/cmp-list.json](https://cmplist.consensu.org/v2/cmp-list.json)
+we want to use our own, we can use an option:
+
+```golang
+err := cmp.NewLoader(cmp.WithURL("https://example.com/cmp-list.json")).LoadIDs()
+```
+
+The format of the JSON must be the same.
+
+If we want to use our own list of valid CMPs we can simply set the variable:
+
+```golang
+cmp.ValidCMPs = []int{1, 123}
+```
+
 ## Testing
 
-We use [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) for testing purposes.
+We use [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/)
+for testing purposes.
 
 ```bash
 ginkgo ./...

benchmark_test.go

@@ -1,68 +1,60 @@
 package iab_tcf
 
 import (
+	"time"
+
 	"github.com/montanaflynn/stats"
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gmeasure"
 )
 
 type data struct {
-	message   string
 	consent string
 	result  string
 }
 
 var _ = Describe("performance", func() {
-
-	const (
-		times = 1000
-	)
-
 	var (
-		testResults = map[string][]int64{}
-		testData    = []data{
-			data{
-				message: "v1 consent",
-				consent: "BOlLbqtOlLbqtAVABADECg-AAAApp7v______9______9uz_Ov_v_f__33e8__9v_l_7_-___u_-3zd4u_1vf99yfm1-7etr3tp_87ues2_Xur__79__3z3_9phP78k89r7337Ew-v02", 
-				result: "111101110111111111111111111111111111111111111111111110111111111111111111111111111111111111111110110111011001111111100111010111111111110111111111101111111111111111111011111011101111011110011111111111111110110111111111110010111111111101111111111111011111111111111111110111011111111111011011111001101110111100010111011111111010110111101111111110111110111001001111110011011010111111011101101111010110110101111011110110110100111111111110011101110111001111010110011011011111101011110111010101111111111111111101111110111111111111111011111001111011111111111110110100110000100111111101111110010010011110011110110101111101111011111011111101100010011000011111010111111010011011",
-			},
-			data{
-				message: "v2 consent",
-				consent: "COxR03kOxR1CqBcABCENAgCMAP_AAH_AAAqIF3EXySoGY2thI2YVFxBEIYwfJxyigMgChgQIsSwNQIeFLBoGLiAAHBGYJAQAGBAEEACBAQIkHGBMCQAAgAgBiRCMQEGMCzNIBIBAggEbY0FACCVmHkHSmZCY7064O__QLuIJEFQMAkSBAIACLECIQwAQDiAAAYAlAAABAhIaAAgIWBQEeAAAACAwAAgAAABBAAACAAQAAICIAAABAAAgAiAQAAAAGgIQAACBABACRIAAAEANCAAgiCEAQg4EAo4AAA", 
-				result: "010001011111001001001010100000011001100011011010110110000100100011011001100001010100010111000100000100010000100001100011000001111100100111000111001010001010000000110010000000001010000110000001000000100010110001001011000000110101000000100001111000010100101100000110100000011000101110001000000000000000011100000100011001100000100100000001000000000000011000000100000000010000010000000000001000000100000001000000100010010000011100011000000100110000001001000000000000000010000000000010000000000110001001000100001000110001000000010000011000110000001011001100110100100000000100100000000100000010000010000000010001101101100011010000010100000000001000001001010110011000011110010000011101001010011001100100001001100011101111010011101011100000111011111111111101",
-			},
+		experiment    *gmeasure.Experiment
+		times         = gmeasure.SamplingConfig{N: 1000}
+		testConsentV1 = data{
+			consent: "BOlLbqtOlLbqtAVABADECg-AAAApp7v______9______9uz_Ov_v_f__33e8__9v_l_7_-___u_-3zd4u_1vf99yfm1-7etr3tp_87ues2_Xur__79__3z3_9phP78k89r7337Ew-v02",
+			result:  "111101110111111111111111111111111111111111111111111110111111111111111111111111111111111111111110110111011001111111100111010111111111110111111111101111111111111111111011111011101111011110011111111111111110110111111111110010111111111101111111111111011111111111111111110111011111111111011011111001101110111100010111011111111010110111101111111110111110111001001111110011011010111111011101101111010110110101111011110110110100111111111110011101110111001111010110011011011111101011110111010101111111111111111101111110111111111111111011111001111011111111111110110100110000100111111101111110010010011110011110110101111101111011111011111101100010011000011111010111111010011011",
+		}
+		testConsentV2 = data{
+			consent: "COxR03kOxR1CqBcABCENAgCMAP_AAH_AAAqIF3EXySoGY2thI2YVFxBEIYwfJxyigMgChgQIsSwNQIeFLBoGLiAAHBGYJAQAGBAEEACBAQIkHGBMCQAAgAgBiRCMQEGMCzNIBIBAggEbY0FACCVmHkHSmZCY7064O__QLuIJEFQMAkSBAIACLECIQwAQDiAAAYAlAAABAhIaAAgIWBQEeAAAACAwAAgAAABBAAACAAQAAICIAAABAAAgAiAQAAAAGgIQAACBABACRIAAAEANCAAgiCEAQg4EAo4AAA",
+			result:  "010001011111001001001010100000011001100011011010110110000100100011011001100001010100010111000100000100010000100001100011000001111100100111000111001010001010000000110010000000001010000110000001000000100010110001001011000000110101000000100001111000010100101100000110100000011000101110001000000000000000011100000100011001100000100100000001000000000000011000000100000000010000010000000000001000000100000001000000100010010000011100011000000100110000001001000000000000000010000000000010000000000110001001000100001000110001000000010000011000110000001011001100110100100000000100100000000100000010000010000000010001101101100011010000010100000000001000001001010110011000011110010000011101001010011001100100001001100011101111010011101011100000111011111111111101",
 		}
 	)
 
-	AfterSuite(func() {
-		for testName, results := range testResults {
-			p99, _ := stats.Percentile(stats.LoadRawData(results), 99)
-			Expect(p99).Should(BeNumerically("<", 2000), testName+" shouldn't take too long.")
-		}
+	BeforeEach(func() {
+		experiment = gmeasure.NewExperiment("consent speed")
 	})
 
-	var run = func(consent string, result string) func(b Benchmarker) {
-		return func(b Benchmarker) {
-			runtime := b.Time("runtime", func() {
+	Context("custom parser", func() {
+		var run = func(consent string, expected string) func(_ int) {
+			return func(_ int) {
 				output, err := NewConsent(consent)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(output.GetConsentBitstring()).To(Equal(result))
-			})
-			testName := CurrentGinkgoTestDescription().TestText
-			testResults[testName] = append(testResults[testName], runtime.Microseconds())
-			b.RecordValue("spent in microseconds", float64(runtime.Microseconds()))
+				Expect(output.GetConsentBitstring()).To(Equal(expected))
+			}
 		}
-	}
-
-	Context("custom parser", func() {
 
-		var runCustom = func(consent string, result string) func(b Benchmarker) {
-			return run(consent, result)
+		var assert = func() {
+			measurements := experiment.Get("runtime")
+			p99, _ := stats.Percentile(stats.LoadRawData(measurements.Durations), 99)
+			Expect(p99).Should(BeNumerically("<", 2000*time.Millisecond), "it shouldn't take too long.")
 		}
 
-		for _, data := range testData {
-			testResults[data.message] = []int64{}
-			Measure(data.message, runCustom(data.consent, data.result), times)
-		}
+		It("is fast with v1", func() {
+			experiment.SampleDuration("runtime", run(testConsentV1.consent, testConsentV1.result), times)
+			assert()
+		})
+
+		It("is fast with v2", func() {
+			experiment.SampleDuration("runtime", run(testConsentV2.consent, testConsentV2.result), times)
+			assert()
+		})
 	})
 })

cmp/consent.go

@@ -0,0 +1,13 @@
+package cmp
+
+type Consent struct{}
+
+// ValidCMPs returns the list of valid CMPs loaded.
+func (c *Consent) ValidCMPs() []int {
+	return ValidCMPs
+}
+
+// IsCMPListLoaded returns if the list of valid CMPs was loaded or not.
+func (c *Consent) IsCMPListLoaded() bool {
+	return c.ValidCMPs() != nil
+}

cmp/consent_test.go

@@ -0,0 +1,40 @@
+package cmp_test
+
+import (
+	"github.com/hybridtheory/iab-tcf/cmp"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Consent", func() {
+	var (
+		testValidCMPs = []int{1, 2, 3}
+		consent       *cmp.Consent
+	)
+
+	BeforeEach(func() {
+		consent = &cmp.Consent{}
+	})
+
+	It("returns valid cmps", func() {
+		cmp.ValidCMPs = testValidCMPs
+		Expect(consent.ValidCMPs()).To(Equal(testValidCMPs))
+	})
+
+	Context("is list loaded", func() {
+		It("returns false if it is not loaded", func() {
+			cmp.ValidCMPs = nil
+			Expect(consent.IsCMPListLoaded()).To(BeFalse())
+		})
+
+		It("returns true if it is loaded but empty", func() {
+			cmp.ValidCMPs = []int{}
+			Expect(consent.IsCMPListLoaded()).To(BeTrue())
+		})
+
+		It("returns true if it is properly loaded", func() {
+			cmp.ValidCMPs = testValidCMPs
+			Expect(consent.IsCMPListLoaded()).To(BeTrue())
+		})
+	})
+})

cmp/loader.go

@@ -0,0 +1,84 @@
+package cmp
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"golang.org/x/exp/maps"
+)
+
+const (
+	DefaultCMPVendorList = "https://cmplist.consensu.org/v2/cmp-list.json"
+)
+
+var (
+	ValidCMPs []int
+)
+
+// Option is the type that allows us to configure the Loader dynamically.
+type Option func(loader *Loader)
+
+// Loader is the type that contains the logic to load and parse a CMP JSON list.
+type Loader struct {
+	URL string
+}
+
+// CMP contains the structure of the CMP info that comes inside the JSON
+type CMP struct {
+	ID           int
+	Name         string
+	IsCommercial bool
+	Environments []string
+}
+
+// WithURL allows to configure a different URL for the CMP JSON list.
+func WithURL(url string) Option {
+	return func(cmp *Loader) {
+		cmp.URL = url
+	}
+}
+
+// NewLoader returns a CMP vendor list loader instance.
+func NewLoader(options ...Option) *Loader {
+	loader := &Loader{
+		URL: DefaultCMPVendorList,
+	}
+	for _, option := range options {
+		option(loader)
+	}
+	return loader
+}
+
+// Unmarshal parses the JSON vendor list into a struct so we can use them.
+func (loader *Loader) Unmarshal(response *http.Response) ([]CMP, error) {
+	type Response struct {
+		CMPS map[string]CMP `json:"cmps"`
+	}
+	data := Response{}
+	if err := json.NewDecoder(response.Body).Decode(&data); err != nil {
+		return []CMP{}, err
+	}
+	return maps.Values(data.CMPS), nil
+}
+
+// Load is used to load the vendor list into a list of CMP information.
+func (loader *Loader) Load() ([]CMP, error) {
+	response, err := http.Get(loader.URL)
+	if err == nil {
+		return loader.Unmarshal(response)
+	}
+	return []CMP{}, err
+}
+
+// LoadIDs loads the list of vendor CMP ids globally so we can reuse it
+// with subsequent calls.
+func (loader *Loader) LoadIDs() error {
+	cmps, err := loader.Load()
+	if err == nil {
+		ValidCMPs = []int{}
+		for _, cmp := range cmps {
+			ValidCMPs = append(ValidCMPs, cmp.ID)
+		}
+	}
+	return err
+}