Merge pull request #384 from opencybersecurityalliance/develop

v1.7.2

CHANGELOG.rst

@@ -9,7 +9,28 @@ The format is based on `Keep a Changelog`_.
 Unreleased
 ==========
 
-- Issue #370 (the last unit test in test_stixshifter_translator.py disabled)
+1.7.2 (2023-07-26)
+==================
+
+Added
+-----
+
+- Minimal version requirements for all dependencies
+- param ``cool_down_after_transmission`` in stix-shifter interface
+- Unit tests on empty input variable for commands
+- ``lark-js`` support for ``kestrel.lark`` #371
+
+Changed
+-------
+
+- Keep stix-shifter to v5 (not v6) to avoid a dependency specification issue
+
+Fixed
+-----
+
+- Fast translation bug on ``group`` keyword in stix-shifter mapping #370
+- ``typeguard`` old version cause exception
+- Exception with empty variable #254
 
 1.7.1 (2023-07-13)
 ==================
@@ -35,10 +56,10 @@ Fixed
 
 - stix-shifter interface translator error msg passing bugs
 - stix-shifter interface transmitter error msg passing bug
-- infinite loop in stix-shifter interface transmitter
+- Infinite loop in stix-shifter interface transmitter
 - stix-shifter connector pip uninstall hanging issue
-- prefetch logic error with empty return
-- dataframe index error in CSV export
+- Prefetch logic error with empty return
+- Dataframe index error in CSV export
 
 1.7.0 (2023-06-14)
 ==================

README.rst

@@ -54,7 +54,7 @@ What does it mean by *hunt fast*?
 - Do NOT write the same TTP pattern in different data source queries.
 - Do NOT write one-time-use adapaters to connect hunt steps.
 - Do NOT waste your existing analytic scripts/programs in future hunts.
-- DD construct your hunt-flow from smaller reuseable hunt-flow.
+- Do construct your hunt-flow from smaller reuseable hunt-flow.
 - Do share your huntbook with your future self and your colleagues.
 - Do get interactive feedback and revise hunt-flow on the fly.
 
@@ -125,6 +125,7 @@ Kestrel Hunting Blogs
 #. `Setting Up The Open Hunting Stack in Hybrid Cloud With Kestrel and SysFlow`_
 #. `Try Kestrel in a Cloud Sandbox`_
 #. `Fun with securitydatasets.com and the Kestrel PowerShell Deobfuscator`_
+#. `Kestrel Data Retrieval Explained`_
 
 Talks And Demos
 ===============
@@ -190,6 +191,7 @@ Connecting With The Community
 .. _Setting Up The Open Hunting Stack in Hybrid Cloud With Kestrel and SysFlow: https://opencybersecurityalliance.org/kestrel-sysflow-open-hunting-stack/
 .. _Try Kestrel in a Cloud Sandbox: https://opencybersecurityalliance.org/try-kestrel-in-a-cloud-sandbox/
 .. _Fun with securitydatasets.com and the Kestrel PowerShell Deobfuscator: https://opencybersecurityalliance.org/fun-with-securitydatasets-com-and-the-kestrel-powershell-deobfuscator/
+.. _Kestrel Data Retrieval Explained: https://opencybersecurityalliance.org/kestrel-data-retrieval-explained/
 
 .. _RSA Conference 2021: https://www.rsaconference.com/Library/presentation/USA/2021/The%20Game%20of%20Cyber%20Threat%20Hunting%20The%20Return%20of%20the%20Fun
 .. _RSA'21 session recording: https://www.youtube.com/watch?v=-Xb086R0JTk

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = kestrel-lang
-version = 1.7.1
+version = 1.7.2
 description = Kestrel Threat Hunting Language
 long_description = file:README.rst
 long_description_content_type = text/x-rst
@@ -27,18 +27,18 @@ package_dir =
 scripts = bin/kestrel
 python_requires = >= 3.8
 install_requires =
-    pyyaml
-    lxml
-    pandas
-    requests
-    nest-asyncio>=1.5.6
+    typeguard>=4.0.0
+    pyyaml>=6.0
+    lxml>=4.9.3
     lark>=1.1.5
+    pandas>=2.0.0
     pyarrow>=5.0.0
     docker>=5.0.0
-    stix-shifter>=5.3.1
-    stix-shifter-utils>=5.3.1
-    firepit>=2.3.24
-    typeguard
+    requests>=2.31.0
+    nest-asyncio>=1.5.6
+    stix-shifter==5.3.1
+    stix-shifter-utils==5.3.1
+    firepit>=2.3.25
 tests_require =
     pytest
 

src/kestrel/codegen/commands.py

@@ -73,15 +73,24 @@ def wrapper(stmt, session):
     return wrapper
 
 
-def _guard_empty_input(func):
+def _skip_command_if_empty_input(func):
     @functools.wraps(func)
     def wrapper(stmt, session):
-        for varname in get_all_input_var_names(stmt):
-            v = session.symtable[varname]
-            if v.length + v.records_count == 0:
-                raise EmptyInputVariable(varname)
-        else:
+        var_names = get_all_input_var_names(stmt)
+        if not var_names:
+            return func(stmt, session)
+        elif any(
+            [
+                session.symtable[v].length + session.symtable[v].records_count
+                for v in var_names
+            ]
+        ):
             return func(stmt, session)
+        elif "output" in stmt:
+            var_struct = new_var(session.store, None, [], stmt, session.symtable)
+            return var_struct, None
+        else:
+            return None, None
 
     return wrapper
 
@@ -102,29 +111,21 @@ def wrapper(stmt, session):
 
 @_debug_logger
 @_default_output
+@_skip_command_if_empty_input
 def assign(stmt, session):
     entity_table = session.symtable[stmt["input"]].entity_table
     transform = stmt.get("transformer")
     if transform:
         qry = _transform_query(session.store, entity_table, transform)
     else:
         qry = Query(entity_table)
-
     qry = _build_query(session.store, entity_table, qry, stmt, [])
-
-    try:
-        session.store.assign_query(stmt["output"], qry)
-        output = new_var(session.store, stmt["output"], [], stmt, session.symtable)
-    except InvalidAttr as e:
-        var_attr = str(e).split()[-1]
-        var_name, _, attr = var_attr.rpartition(".")
-        raise MissingEntityAttribute(var_name, attr) from e
-
-    return output, None
+    session.store.assign_query(stmt["output"], qry)
 
 
 @_debug_logger
 @_default_output
+@_skip_command_if_empty_input
 def merge(stmt, session):
     entity_types = list(
         set([session.symtable[var_name].type for var_name in stmt["inputs"]])
@@ -134,9 +135,8 @@ def merge(stmt, session):
     entity_tables = [
         session.symtable[var_name].entity_table for var_name in stmt["inputs"]
     ]
+    entity_tables = [t for t in entity_tables if t is not None]
     session.store.merge(stmt["output"], entity_tables)
-    output = new_var(session.store, stmt["output"], [], stmt, session.symtable)
-    return output, None
 
 
 @_debug_logger
@@ -154,7 +154,6 @@ def load(stmt, session):
 
 
 @_debug_logger
-@_guard_empty_input
 def save(stmt, session):
     dump_data_to_file(
         session.store, session.symtable[stmt["input"]].entity_table, stmt["path"]
@@ -163,6 +162,7 @@ def save(stmt, session):
 
 
 @_debug_logger
+@_skip_command_if_empty_input
 def info(stmt, session):
     header = session.store.columns(session.symtable[stmt["input"]].entity_table)
     direct_attrs, associ_attrs, custom_attrs, references = [], [], [], []
@@ -202,6 +202,7 @@ def info(stmt, session):
 
 
 @_debug_logger
+@_skip_command_if_empty_input
 def disp(stmt, session):
     entity_table = session.symtable[stmt["input"]].entity_table
     transform = stmt.get("transformer")
@@ -224,6 +225,7 @@ def disp(stmt, session):
 
 @_debug_logger
 @_default_output
+@_skip_command_if_empty_input
 def get(stmt, session):
     pattern = stmt["stixpattern"]
     local_var_table = stmt["output"] + "_local"
@@ -296,7 +298,7 @@ def get(stmt, session):
 
 @_debug_logger
 @_default_output
-@_guard_empty_input
+@_skip_command_if_empty_input
 def find(stmt, session):
     # shortcuts
     return_type = stmt["type"]
@@ -344,7 +346,7 @@ def find(stmt, session):
 
 @_debug_logger
 @_default_output
-@_guard_empty_input
+@_skip_command_if_empty_input
 def join(stmt, session):
     session.store.join(
         stmt["output"],
@@ -357,7 +359,7 @@ def join(stmt, session):
 
 @_debug_logger
 @_default_output
-@_guard_empty_input
+@_skip_command_if_empty_input
 def group(stmt, session):
     if "aggregations" in stmt:
         aggs = [(i["func"], i["attr"], i["alias"]) for i in stmt["aggregations"]]
@@ -373,7 +375,7 @@ def group(stmt, session):
 
 @_debug_logger
 @_default_output
-@_guard_empty_input
+@_skip_command_if_empty_input
 def sort(stmt, session):
     session.store.assign(
         stmt["output"],
@@ -386,7 +388,7 @@ def sort(stmt, session):
 
 @_debug_logger
 @_default_output
-@_guard_empty_input
+@_skip_command_if_empty_input
 def apply(stmt, session):
     arg_vars = [session.symtable[v_name] for v_name in stmt["inputs"]]
     display = session.analytics_manager.execute(

src/kestrel/codegen/summary.py

@@ -11,7 +11,7 @@
 )
 from collections import OrderedDict
 from kestrel.codegen.relations import get_entity_id_attribute
-from kestrel.exceptions import KestrelInternalError
+from kestrel.exceptions import KestrelInternalError, MissingEntityAttribute
 
 
 def gen_variable_summary(var_name, var_struct):
@@ -92,7 +92,13 @@ def get_variable_entity_count(variable):
     entity_count = 0
     if variable.entity_table:
         entity_id_attr = get_entity_id_attribute(variable)
-        if entity_id_attr not in variable.store.columns(variable.entity_table):
-            return 0
-        entity_count = variable.store.count(variable.entity_table)
+        try:
+            columns = variable.store.columns(variable.entity_table)
+        except InvalidAttr as e:
+            # TODO: a better solution needed for tests/test_timestamped.py::test_timestamped_grouped_assign
+            table_attr = str(e).split()[-1]
+            table_name, _, attr = table_attr.rpartition(".")
+            raise MissingEntityAttribute(table_name, attr) from e
+        if entity_id_attr in columns:
+            entity_count = variable.store.count(variable.entity_table)
     return entity_count