Simplify TIToken reprs

tivars/token.py

@@ -20,25 +20,48 @@ def __init__(self, token: Token):
         self.translation = self.langs[None] = self.langs["en"]
 
     def __repr__(self) -> str:
-        return f"<{self.display} ({self.escape})>"
+        return self.escape
+
+    def __str__(self) -> str:
+        return self.display
 
     @property
     def accessible(self) -> str:
+        """
+        :return: The accessible name of this token in English (``en``)
+        """
+
         return self.translation.accessible
 
     @property
     def display(self) -> str:
+        """
+        :return: The display name of this token in English (``en``)
+        """
+
         return self.translation.display
 
     @property
     def escape(self) -> str:
+        """
+        :return: The escape sequence for this token as a byte literal
+        """
+
         return rf"\{'x' if len(self.bits) == 1 else 'u'}{self.bits.hex()}"
 
     def names(self) -> list[str]:
+        """
+        :return: The names of this token in English (``en``)
+        """
+
         return self.translation.names()
 
 
 class IllegalToken(TIToken):
+    """
+    `TIToken` subclass for denoting illegal (i.e. nonexistent) tokens
+    """
+
     def __init__(self, bits: bytes):
         self.bits = bits
 

tivars/tokenizer/state.py

@@ -98,7 +98,7 @@ class MinMode(EncoderState):
 
 class Line(EncoderState):
     """
-    Encoder state which is always exited after a line break
+    Encoder state which is always exited after a line break or STO
     """
 
     def next(self, token: TIToken) -> list[EncoderState]:
@@ -193,6 +193,7 @@ class SmartMode(EncoderState):
 
     def next(self, token: TIToken) -> list[EncoderState]:
         match token.bits:
+            #    "
             case b'\x2A':
                 return [self, String()]