Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Linked checksum #135

Merged
merged 5 commits into from
Dec 4, 2024
Merged

Linked checksum #135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cb512e1
First implementation of the hash based link logic
CsatariGergely Dec 4, 2024
4fc8830
Merge branch 'main' into linked-checksum
CsatariGergely Dec 4, 2024
7c608fd
Support for link detection based on checksums
CsatariGergely Dec 4, 2024
af7db91
Removal of log file
CsatariGergely Dec 4, 2024
9174528
Fixing deduplication
CsatariGergely Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions tools/openchain_telco_sbom_validator/setup.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ classifiers =
[options]
package_dir =
= src
packages = find:
python_requires = >=3.0
packages = find_namespace:
include_package_data = True
python_requires = >=3.3
zip_safe = True
install_requires =
spdx-tools>=0.8.2
Expand Down
20 changes: 15 additions & 5 deletions tools/openchain_telco_sbom_validator/src/openchain_telco_sbom_validator/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import argparse
import logging
import sys
from spdx_tools.spdx.model.document import Document

from openchain_telco_sbom_validator.validator import Validator
from openchain_telco_sbom_validator.reporter import reportCli,reportVersion
Expand All @@ -26,7 +25,13 @@ def main():
else:
logLevel = logging.INFO

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logLevel)
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(message)s',
level=logLevel,
handlers=[
logging.FileHandler("log.log"), # Write logs to a file
logging.StreamHandler()
])

if args.version:
reportVersion()
Expand All @@ -36,6 +41,10 @@ def main():

filePath = str(args.input)
validator = Validator()

reference_logic = args.reference_logic
if None == reference_logic:
reference_logic = "none"
result, problems = validator.validate(filePath,
args.strict_purl_check,
args.strict_url_check,
Expand Down Expand Up @@ -90,9 +99,10 @@ def parseArguments(additionalArguments: AdditionalArguments = AdditionalArgument
parser.add_argument('--reference-logic',
help='Defines the logic how the referenced files are accessible. If not added the referenced'
'files will not be investigated.'
'Built in supported logics are none (no linked files are investigated) yocto-all (all external'
'refs are investigated) and yocto-contains-only (only those files are investigated which are in'
'CONTAIN relatioships). It is possible to register more reference logics in library mode')
'Built in supported logics are none (no linked files are investigated) yocto-all (all external '
'refs are investigated), yocto-contains-only (only those files are investigated which are in '
'CONTAIN relatioships) and checksum-all (files are identified based on their checksum). It is '
'possible to register more reference logics in library mode')
parser.add_argument('--version', action="store_true",
help='Prints version and exits',
required=False)
Expand Down
72 changes: 72 additions & 0 deletions tools/openchain_telco_sbom_validator/src/openchain_telco_sbom_validator/validator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from spdx_tools.spdx.parser.error import SPDXParsingError
from spdx_tools.spdx.model.package import ExternalPackageRefCategory
from spdx_tools.spdx.model.relationship import RelationshipType
from spdx_tools.spdx.model.checksum import ChecksumAlgorithm
from spdx_tools.spdx import document_utils
from packageurl.contrib import purl2url
import ntia_conformance_checker as ntia
Expand Down Expand Up @@ -118,10 +119,12 @@ def _dummy_function_package(self, problems: Problems, package: Package):
class Validator:

def __init__(self):
self.referringLogicStore = {}
self.referringLogics = {}
self.addReferringLogics("none", referred_none)
self.addReferringLogics("yocto-all", referred_yocto_all)
self.addReferringLogics("yocto-contains-only", referred_yocto_contains_only)
self.addReferringLogics("checksum-all", referred_checksum_all)

return None

Expand Down Expand Up @@ -451,6 +454,75 @@ def referred_yocto_contains_only(self, doc: Document, dir_name: str):
documents.append(external_refs[spdx_document_id])
return documents

def referred_checksum_all(self, doc: Document, dir_name: str):
# Known limitations: MD2, MD4 and MD6 hashes are not supported
import hashlib
from pathlib import Path
logger.debug("In Yocto contains only")
documents = []
checksums = {}
algorithms = {}
if doc.creation_info.external_document_refs:
logger.debug(f"--------------We have refs!------------")
for ref in doc.creation_info.external_document_refs:
logger.debug(f"SPDX document referenced {ref.document_uri}, {ref.checksum.algorithm}, {ref.checksum.value}")
if ref.checksum.algorithm in self.referringLogicStore.keys():
if ref.checksum.value in self.referringLogicStore[ref.checksum.algorithm].keys():
logger.debug(f"Document found in LogicStore {ref.checksum.algorithm}, {ref.checksum.value}")
documents.append(self.referringLogicStore[ref.checksum.algorithm][ref.checksum.value])
if not ref.checksum.algorithm in checksums.keys():
checksums[ref.checksum.algorithm] = []
checksums[ref.checksum.algorithm].append(ref.checksum.value)
algorithms[ref.checksum.algorithm] = None
for algorithm in algorithms.keys():
for spdx_file in [f.name for f in Path(dir_name).iterdir() if f.is_file()]:
logger.debug(f"Calculating {algorithm} hash for {dir_name}, {spdx_file}")
if dir_name == "":
doc_location = spdx_file
else:
doc_location = f"{dir_name}/{spdx_file}"
logger.debug(f"Document location is: {doc_location}")
with open(doc_location, 'rb') as f:
hash = None
# SHA1, SHA224, SHA256, SHA384, SHA512, MD2, MD4, MD5, MD6
match algorithm:
case ChecksumAlgorithm.SHA1:
hash = hashlib.sha1()
case ChecksumAlgorithm.SHA224:
hash = hashlib.sha224()
case ChecksumAlgorithm.SHA256:
hash = hashlib.sha256()
case ChecksumAlgorithm.SHA384:
hash = hashlib.sha384()
case ChecksumAlgorithm.SHA512:
hash = hashlib.sha512()
case ChecksumAlgorithm.MD5:
hash = hashlib.md5()
case _:
logger.error(f"{algorithm} is not supported.")

while chunk := f.read(8192):
hash.update(chunk)
logger.debug(f"Storing file information for {algorithm}, {doc_location}, {hash.name}, {hash.hexdigest()}")
if algorithm not in self.referringLogicStore:
self.referringLogicStore[algorithm] = {}
self.referringLogicStore[algorithm][hash.hexdigest()] = doc_location
for algorithm in checksums.keys():
for checksum in checksums[algorithm]:
logger.debug(f"Getting information from Logic Store for {algorithm}, {checksum}")
if algorithm in self.referringLogicStore:
if checksum in self.referringLogicStore[algorithm]:
documents.append(self.referringLogicStore[algorithm][checksum])
else:
logger.error(f"Checksum not found in Logic Store {algorithm}, {checksum}")
else:
logger.error(f"Algorithm not found in Logic Store {algorithm}, {checksum}")

documents_dd = set()
documents_dd = [x for x in documents if not (x in documents_dd or documents_dd.add(x))]

return documents_dd

def referred_none(self, doc: Document, dir_name: str):
return []

Expand Down
2 changes: 1 addition & 1 deletion tools/openchain_telco_sbom_validator/testing/linked-sboms-01/alarm.spdx.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "9642680ac03cefca366190172674968a6bf1e0bc"
"checksumValue": "bbae9c1da0df80bcc95e27ba0ff8c05de851f967"
},
"externalDocumentId": "DocumentRef-recipe-alarm",
"spdxDocument": "http://spdx.org/spdxdoc/recipe-alarm-7437445b-18f5-5395-8472-5062f4bd2455"
Expand Down
2 changes: 1 addition & 1 deletion tools/openchain_telco_sbom_validator/testing/linked-sboms-01/alignmentpavendors.spdx.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "2bcd0cd44fe211f6b9438cce2d68cd793055bc23"
"checksumValue": "23b6232efdd7364eee3ae7b86aa2aafb8cbc970d"
},
"externalDocumentId": "DocumentRef-recipe-alignmentpavendors",
"spdxDocument": "http://spdx.org/spdxdoc/recipe-alignmentpavendors-fe777936-b492-5e26-a994-a99932af7af9"
Expand Down
2 changes: 1 addition & 1 deletion tools/openchain_telco_sbom_validator/testing/linked-sboms-01/alps.spdx.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"checksums": [
{
"algorithm": "SHA1",
"checksumValue": "9994e47d3d6780110d7e70741db45146aae2bd4a"
"checksumValue": "5be5deb519512f883b127648927439d4ab3c53a3"
},
{
"algorithm": "SHA256",
Expand Down
16 changes: 8 additions & 8 deletions tools/openchain_telco_sbom_validator/testing/linked-sboms-01/linked-sbom-01.spdx.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,63 +16,63 @@
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "284bc7e009c05041f4d82d831cc4ea83c14f89cc"
"checksumValue": "0f2c1460eb555b50402c9a2d14719ea2cfe7e667"
},
"externalDocumentId": "DocumentRef-alarm",
"spdxDocument": "http://spdx.org/spdxdoc/alarm-a89a4665-0835-5596-9956-17f2a0fd5562"
},
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "9667d7e0998906c0e19254e068a8e25211468a95"
"checksumValue": "9db8ef443f76453a2d4539958d58362d661374b3"
},
"externalDocumentId": "DocumentRef-runtime-alarm",
"spdxDocument": "http://spdx.org/spdxdoc/runtime-alarm-d99ec519-43f0-5e6f-a178-d50df7c99645"
},
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "56aed1db68ab1ecf46fdd4f0ad874c65c0ea85e8"
"checksumValue": "e596b87aedf271e7319846cf26fb94ef4c9eb74f"
},
"externalDocumentId": "DocumentRef-alignmentpavendors",
"spdxDocument": "http://spdx.org/spdxdoc/alignmentpavendors-a7cf1096-d17a-5e0a-b3d8-2310355c80a3"
},
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "d65734d247e6aaf7be6463e9742ac339db547e8c"
"checksumValue": "79944b521186c2ab8ffbc7fc81542d8f2d7cb919"
},
"externalDocumentId": "DocumentRef-runtime-alignmentpavendors",
"spdxDocument": "http://spdx.org/spdxdoc/runtime-alignmentpavendors-90c55062-7c92-5fea-b6f9-fafe2b419839"
},
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "2d8900895d257e501512f4ff7734021a2a598610"
"checksumValue": "d32061198c7cad06e65e07aebd315d4a03478b32"
},
"externalDocumentId": "DocumentRef-alps",
"spdxDocument": "http://spdx.org/spdxdoc/alps-717e3bbe-606c-55de-a47f-f82c29bce191"
},
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "1dfed264d706ec5c816c07b25d33237ac4a78ac7"
"checksumValue": "095263fb1a73332b90ce197421866fb68ddfea39"
},
"externalDocumentId": "DocumentRef-runtime-alps",
"spdxDocument": "http://spdx.org/spdxdoc/runtime-alps-5a729a35-3b9b-5f22-8516-ce0c299cbec1"
},
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "26e30638da946d0283486647268bfb6373f3834f"
"checksumValue": "210a99859bb58709beef987a8d0c098bdaea407d"
},
"externalDocumentId": "DocumentRef-kernel-5.15.155-nokia-r42",
"spdxDocument": "http://spdx.org/spdxdoc/kernel-5.15.155-r42-449e38e4-d16a-5c20-b592-d602fdc832df"
},
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "19280f249c636e5a23cc7b9e92abfd8c5124e195"
"checksumValue": "57f4565c3392ee4e147ee0c0b4716a06a3cd1a84"
},
"externalDocumentId": "DocumentRef-runtime-kernel-5.15.155-nokia-r42",
"spdxDocument": "http://spdx.org/spdxdoc/runtime-kernel-5.15.155-r42-98fa9061-a335-5876-b6cb-cbb0661d1328"
Expand Down
4 changes: 2 additions & 2 deletions tools/openchain_telco_sbom_validator/testing/linked-sboms-01/runtime-alarm.spdx.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "284bc7e009c05041f4d82d831cc4ea83c14f89cc"
"checksumValue": "0f2c1460eb555b50402c9a2d14719ea2cfe7e667"
},
"externalDocumentId": "DocumentRef-package-alarm",
"spdxDocument": "http://spdx.org/spdxdoc/alarm-a89a4665-0835-5596-9956-17f2a0fd5562"
},
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "a13ea50dde972f54747a90838e8d383d27dc3cf8"
"checksumValue": "87e913c85676eabd22059b5a25fc587c0cb00dab"
},
"externalDocumentId": "DocumentRef-runtime-dependency-em-accessories",
"spdxDocument": "http://spdx.org/spdxdoc/em-accessories-53e18417-f793-5345-aa0e-24f7c9b4f8c8"
Expand Down
2 changes: 1 addition & 1 deletion ...enchain_telco_sbom_validator/testing/linked-sboms-01/runtime-alignmentpavendors.spdx.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "56aed1db68ab1ecf46fdd4f0ad874c65c0ea85e8"
"checksumValue": "e596b87aedf271e7319846cf26fb94ef4c9eb74f"
},
"externalDocumentId": "DocumentRef-package-alignmentpavendors",
"spdxDocument": "http://spdx.org/spdxdoc/alignmentpavendors-a7cf1096-d17a-5e0a-b3d8-2310355c80a3"
Expand Down
11 changes: 1 addition & 10 deletions ...nchain_telco_sbom_validator/testing/linked-sboms-01/runtime-kernel-5.15.155-r42.spdx.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,7 @@
},
"dataLicense": "CC0-1.0",
"documentNamespace": "http://spdx.org/spdxdoc/runtime-kernel-5.15.155-nokia-r42-98fa9061-a335-5876-b6cb-cbb0661d1328",
"externalDocumentRefs": [
{
"checksum": {
"algorithm": "SHA1",
"checksumValue": "26e30638da946d0283486647268bfb6373f3834f"
},
"externalDocumentId": "DocumentRef-package-kernel-base",
"spdxDocument": "http://spdx.org/spdxdoc/kernel-5.15.155-r42-449e38e4-d16a-5c20-b592-d602fdc832df"
}
],
"externalDocumentRefs": [],
"name": "runtime-kernel-5.15.155-nokia-r42",
"relationships": [
{
Expand Down
16 changes: 14 additions & 2 deletions tools/openchain_telco_sbom_validator/testing/test-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,8 +176,8 @@ function test_linked_yocto_all
assert_terminated_normally
assert_exit_fail
assert_has_output
assert_output_contains "37 | SPDX"
assert_output_contains "One or more of the SPDX files linked-sbom-01.spdx.json, alarm.spdx.json, recipe-alarm.spdx.json, runtime-alarm.spdx.json, alarm.spdx.json, recipe-alarm.spdx.json, em-accessories.spdx.json, alignmentpavendors.spdx.json, recipe-alignmentpavendors.spdx.json, runtime-alignmentpavendors.spdx.json, alignmentpavendors.spdx.json, recipe-alignmentpavendors.spdx.json, alps.spdx.json, runtime-alps.spdx.json, kernel-5.15.155-r42.spdx.json, runtime-kernel-5.15.155-r42.spdx.json, kernel-5.15.155-r42.spdx.json are not compliant with the OpenChain Telco SBOM Guide"
assert_output_contains "42 | SPDX"
assert_output_contains "One or more of the SPDX files linked-sbom-01.spdx.json, alarm.spdx.json, recipe-alarm.spdx.json, runtime-alarm.spdx.json, alarm.spdx.json, recipe-alarm.spdx.json, em-accessories.spdx.json, alignmentpavendors.spdx.json, recipe-alignmentpavendors.spdx.json, runtime-alignmentpavendors.spdx.json, alignmentpavendors.spdx.json, recipe-alignmentpavendors.spdx.json, alps.spdx.json, runtime-alps.spdx.json, kernel-5.15.155-r42.spdx.json, runtime-kernel-5.15.155-r42.spdx.json are not compliant with the OpenChain Telco SBOM Guide"
}

function test_linked_yocto-contains-only
Expand All @@ -204,6 +204,18 @@ function test_linked_nonexistent
assert_output_contains "The SPDX file linked-sboms-01/linked-sbom-01.spdx.json is not compliant with the OpenChain Telco SBOM Guide"
}

function test_linked_checksum_all
{
echo "Test: test_linked_checksum_all"
run "openchain-telco-sbom-validator --reference-logic checksum-all linked-sboms-01/linked-sbom-01.spdx.json"
echo "$output"
assert_terminated_normally
assert_exit_fail
assert_has_output
assert_output_contains "42 | SPDX"
assert_output_contains "One or more of the SPDX files linked-sbom-01.spdx.json, alarm.spdx.json, recipe-alarm.spdx.json, runtime-alarm.spdx.json, alarm.spdx.json, recipe-alarm.spdx.json, em-accessories.spdx.json, alignmentpavendors.spdx.json, recipe-alignmentpavendors.spdx.json, runtime-alignmentpavendors.spdx.json, alignmentpavendors.spdx.json, recipe-alignmentpavendors.spdx.json, alps.spdx.json, runtime-alps.spdx.json, kernel-5.15.155-r42.spdx.json, runtime-kernel-5.15.155-r42.spdx.json are not compliant with the OpenChain Telco SBOM Guide"
}



testrunner
Loading