Fix null byte detokenization

TI-Toolkit · Jan 3, 2024 · 5f6bf80 · 5f6bf80
1 parent e9f9176
commit 5f6bf80
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/tivars/tokenizer/__init__.py b/tivars/tokenizer/__init__.py
@@ -35,6 +35,9 @@ def decode(bytestream: bytes, *,
                 else:
                     raise ValueError(f"unrecognized bytes '0x{curr_bytes[0]:x}{curr_bytes[1]:x}' at position {index}")
 
+        else:
+            curr_bytes = b''
+
         index += 1
 
     return b''.join(out) if mode == "ti_ascii" else "".join(out), since