From 5f4c80e4e8d220fd45d08ea9e8404c0e10762795 Mon Sep 17 00:00:00 2001 From: Paul Coccoli Date: Fri, 25 Feb 2022 14:32:09 -0500 Subject: [PATCH 1/7] Don't prepend tmpdir when local_storage_path is a URI --- src/kestrel/session.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/kestrel/session.py b/src/kestrel/session.py index 7ae5ab62..4820477c 100644 --- a/src/kestrel/session.py +++ b/src/kestrel/session.py @@ -225,12 +225,16 @@ def __init__( runtime_directory_master.unlink() runtime_directory_master.symlink_to(self.runtime_directory) - # local database of SQLite or Parquet + # local database of SQLite or PostgreSQL if not store_path: # use the default local database in config.py - store_path = os.path.join( - self.runtime_directory, self.config["session"]["local_database_path"] - ) + local_database_path = self.config["session"]["local_database_path"] + if "://" in local_database_path: + store_path = local_database_path + else: + store_path = os.path.join( + self.runtime_directory, local_database_path + ) self.store = get_storage(store_path, self.session_id) # Symbol Table From 576f09707a55fc1f3a0285cbf6ec05cb0fb3c20c Mon Sep 17 00:00:00 2001 From: Paul Coccoli Date: Fri, 25 Feb 2022 14:50:06 -0500 Subject: [PATCH 2/7] Formatting fix --- src/kestrel/session.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/kestrel/session.py b/src/kestrel/session.py index 4820477c..c1117e33 100644 --- a/src/kestrel/session.py +++ b/src/kestrel/session.py @@ -232,9 +232,7 @@ def __init__( if "://" in local_database_path: store_path = local_database_path else: - store_path = os.path.join( - self.runtime_directory, local_database_path - ) + store_path = os.path.join(self.runtime_directory, local_database_path) self.store = get_storage(store_path, self.session_id) # Symbol Table From dd1e083cd7740a9766eb017fa0fdc76b1d509687 Mon Sep 17 00:00:00 2001 From: Paul Coccoli Date: Fri, 25 Feb 2022 15:23:14 -0500 Subject: [PATCH 3/7] Add unit test for APPLY after GET --- tests/test_python_analytics.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_python_analytics.py b/tests/test_python_analytics.py index 31042390..fc626b29 100644 --- a/tests/test_python_analytics.py +++ b/tests/test_python_analytics.py @@ -6,6 +6,12 @@ from kestrel.codegen.display import DisplayHtml +@pytest.fixture +def fake_bundle_file(): + cwd = os.path.dirname(os.path.abspath(__file__)) + return os.path.join(cwd, "test_bundle.json") + + @pytest.fixture(autouse=True) def env_setup(tmp_path): @@ -94,3 +100,16 @@ def test_enrich_multiple_variables(): assert set([v3[0]["x_new_attr"], v3[1]["x_new_attr"]]) == set( ["newval_c0", "newval_c1"] ) + + +def test_enrich_after_get(fake_bundle_file): + with Session() as s: + stmt = f""" +newvar = get url from file://{fake_bundle_file} where [url:value LIKE '%'] +APPLY python://enrich_one_variable ON newvar +""" + s.execute(stmt) + v = s.get_variable("newvar") + assert len(v) == 31 + assert v[0]["type"] == "url" + assert "x_new_attr" in v[0] From a6ea4f60ad0ad86588975ada652b81a451eee40f Mon Sep 17 00:00:00 2001 From: Paul Coccoli Date: Mon, 28 Feb 2022 15:36:23 -0500 Subject: [PATCH 4/7] Require firepit>=1.3.5 to fix transaction errors --- setup.cfg | 2 +- tests/test_python_analytics.py | 24 ++++++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index 28ebf76a..c65a406e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ install_requires = docker>=5.0.0 stix-shifter>=3.6.0 stix-shifter-utils>=3.6.0 - firepit>=1.3.0, <2.0.0 + firepit>=1.3.5, <2.0.0 tests_require = pytest diff --git a/tests/test_python_analytics.py b/tests/test_python_analytics.py index fc626b29..df8dcda5 100644 --- a/tests/test_python_analytics.py +++ b/tests/test_python_analytics.py @@ -12,6 +12,12 @@ def fake_bundle_file(): return os.path.join(cwd, "test_bundle.json") +@pytest.fixture +def fake_bundle_4(): + cwd = os.path.dirname(os.path.abspath(__file__)) + return os.path.join(cwd, "test_bundle_4.json") + + @pytest.fixture(autouse=True) def env_setup(tmp_path): @@ -71,9 +77,6 @@ def test_html_visualization(): assert viz.html == "

Hello World! -- a Kestrel analytics

" -@pytest.mark.skip( - reason="to fix: multiple variables reassign in APPLY gives a firepit exception" -) def test_enrich_multiple_variables(): with Session() as s: stmt = """ @@ -102,7 +105,7 @@ def test_enrich_multiple_variables(): ) -def test_enrich_after_get(fake_bundle_file): +def test_enrich_after_get_url(fake_bundle_file): with Session() as s: stmt = f""" newvar = get url from file://{fake_bundle_file} where [url:value LIKE '%'] @@ -113,3 +116,16 @@ def test_enrich_after_get(fake_bundle_file): assert len(v) == 31 assert v[0]["type"] == "url" assert "x_new_attr" in v[0] + + +def test_enrich_after_get_process(fake_bundle_4): + with Session() as s: + stmt = f""" +newvar = get process from file://{fake_bundle_4} where [process:binary_ref.name LIKE '%'] +APPLY python://enrich_one_variable ON newvar +""" + s.execute(stmt) + v = s.get_variable("newvar") + assert len(v) == 4 + assert v[0]["type"] == "process" + assert "x_new_attr" in v[0] From ee24f998654bddc0ad2e001e54ab37de924ca1a5 Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Wed, 2 Mar 2022 11:38:04 -0500 Subject: [PATCH 5/7] bug fix: verify_package_origin() takes 1 argument --- src/kestrel_datasource_stixshifter/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kestrel_datasource_stixshifter/interface.py b/src/kestrel_datasource_stixshifter/interface.py index ba204d63..ed4ba02e 100644 --- a/src/kestrel_datasource_stixshifter/interface.py +++ b/src/kestrel_datasource_stixshifter/interface.py @@ -162,7 +162,7 @@ def check_module_availability(connector_name): package_name = get_package_name(connector_name) _logger.debug(f"guess the connector package name: {package_name}") - verify_package_origin(connector_name, package_name) + verify_package_origin(connector_name) _logger.info(f'install Python package "{package_name}".') try: From 740c9595cf8b209e27c57a8563ead95338dc5e16 Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Wed, 2 Mar 2022 11:49:29 -0500 Subject: [PATCH 6/7] remove Python 3.6 (EOL) from testing env --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 4812a96d..ddb2d59f 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -22,7 +22,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ['3.7', '3.8', '3.9'] steps: - uses: actions/checkout@v2 - name: Set up Python From 323c3c3c765b3822779538ff51e74381ec294e6c Mon Sep 17 00:00:00 2001 From: Xiaokui Shu Date: Wed, 2 Mar 2022 11:55:13 -0500 Subject: [PATCH 7/7] v1.2.2 --- CHANGELOG.rst | 21 +++++++++++++++++++++ setup.cfg | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9aa0006b..32d55982 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,27 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog`_. +1.2.2 (2022-03-02) +================== + +Added +----- + +- remote data store support +- unit test: Python analytics: APPLY after GET +- unit test: Python analytics: APPLY on multiple variables + +Fixed +----- + +- bump firepit version to fix transaction errors +- bug fix: verify_package_origin() takes 1 argument + +Removed +------- + +- unit test: Python 3.6 EOL and removed from GitHub Actions + 1.2.1 (2022-02-24) ================== diff --git a/setup.cfg b/setup.cfg index c65a406e..f6d71df8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = kestrel-lang -version = 1.2.1 +version = 1.2.2 description = Kestrel Threat Hunting Language long_description = file:README.rst long_description_content_type = text/x-rst