Merge pull request #179 from opencybersecurityalliance/develop

v1.2.2

.github/workflows/unit-testing.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.6', '3.7', '3.8', '3.9']
+        python-version: ['3.7', '3.8', '3.9']
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python

CHANGELOG.rst

@@ -6,6 +6,27 @@ All notable changes to this project will be documented in this file.
 
 The format is based on `Keep a Changelog`_.
 
+1.2.2 (2022-03-02)
+==================
+
+Added
+-----
+
+- remote data store support
+- unit test: Python analytics: APPLY after GET
+- unit test: Python analytics: APPLY on multiple variables
+
+Fixed
+-----
+
+- bump firepit version to fix transaction errors
+- bug fix: verify_package_origin() takes 1 argument
+
+Removed
+-------
+
+- unit test: Python 3.6 EOL and removed from GitHub Actions
+
 1.2.1 (2022-02-24)
 ==================
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = kestrel-lang
-version = 1.2.1
+version = 1.2.2
 description = Kestrel Threat Hunting Language
 long_description = file:README.rst
 long_description_content_type = text/x-rst
@@ -37,7 +37,7 @@ install_requires =
     docker>=5.0.0
     stix-shifter>=3.6.0
     stix-shifter-utils>=3.6.0
-    firepit>=1.3.0, <2.0.0
+    firepit>=1.3.5, <2.0.0
 tests_require =
     pytest
 

src/kestrel/session.py

@@ -225,12 +225,14 @@ def __init__(
                 runtime_directory_master.unlink()
             runtime_directory_master.symlink_to(self.runtime_directory)
 
-        # local database of SQLite or Parquet
+        # local database of SQLite or PostgreSQL
         if not store_path:
             # use the default local database in config.py
-            store_path = os.path.join(
-                self.runtime_directory, self.config["session"]["local_database_path"]
-            )
+            local_database_path = self.config["session"]["local_database_path"]
+            if "://" in local_database_path:
+                store_path = local_database_path
+            else:
+                store_path = os.path.join(self.runtime_directory, local_database_path)
         self.store = get_storage(store_path, self.session_id)
 
         # Symbol Table

src/kestrel_datasource_stixshifter/interface.py

@@ -162,7 +162,7 @@ def check_module_availability(connector_name):
         package_name = get_package_name(connector_name)
         _logger.debug(f"guess the connector package name: {package_name}")
 
-        verify_package_origin(connector_name, package_name)
+        verify_package_origin(connector_name)
 
         _logger.info(f'install Python package "{package_name}".')
         try:

tests/test_python_analytics.py

@@ -6,6 +6,18 @@
 from kestrel.codegen.display import DisplayHtml
 
 
+@pytest.fixture
+def fake_bundle_file():
+    cwd = os.path.dirname(os.path.abspath(__file__))
+    return os.path.join(cwd, "test_bundle.json")
+
+
+@pytest.fixture
+def fake_bundle_4():
+    cwd = os.path.dirname(os.path.abspath(__file__))
+    return os.path.join(cwd, "test_bundle_4.json")
+
+
 @pytest.fixture(autouse=True)
 def env_setup(tmp_path):
 
@@ -65,9 +77,6 @@ def test_html_visualization():
         assert viz.html == "<p>Hello World! -- a Kestrel analytics</p>"
 
 
-@pytest.mark.skip(
-    reason="to fix: multiple variables reassign in APPLY gives a firepit exception"
-)
 def test_enrich_multiple_variables():
     with Session() as s:
         stmt = """
@@ -94,3 +103,29 @@ def test_enrich_multiple_variables():
         assert set([v3[0]["x_new_attr"], v3[1]["x_new_attr"]]) == set(
             ["newval_c0", "newval_c1"]
         )
+
+
+def test_enrich_after_get_url(fake_bundle_file):
+    with Session() as s:
+        stmt = f"""
+newvar = get url from file://{fake_bundle_file} where [url:value LIKE '%']
+APPLY python://enrich_one_variable ON newvar
+"""
+        s.execute(stmt)
+        v = s.get_variable("newvar")
+        assert len(v) == 31
+        assert v[0]["type"] == "url"
+        assert "x_new_attr" in v[0]
+
+
+def test_enrich_after_get_process(fake_bundle_4):
+    with Session() as s:
+        stmt = f"""
+newvar = get process from file://{fake_bundle_4} where [process:binary_ref.name LIKE '%']
+APPLY python://enrich_one_variable ON newvar
+"""
+        s.execute(stmt)
+        v = s.get_variable("newvar")
+        assert len(v) == 4
+        assert v[0]["type"] == "process"
+        assert "x_new_attr" in v[0]