From 47f8e9dcebf4189064703103f7350fe6071971be Mon Sep 17 00:00:00 2001
From: Sebastiaan Mathot <s.mathot@cogsci.nl>
Date: Fri, 15 Nov 2024 13:36:47 +0100
Subject: [PATCH] Make parsing more robust to issues with triggers - Ghost
 triggers are removed (sent 1 ms before another trigger) - Duplicate event
 triggers are removed (the 1st one is kept) - If the first trial triggers are
 missing, the EEG data is cropped as well (this happens when the EEG recording
 is triggered too late)

---
 eeg_eyetracking_parser/_parsing.py  | 18 +++++++++++++-----
 eeg_eyetracking_parser/_triggers.py | 29 ++++++++++++++++++++++-------
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/eeg_eyetracking_parser/_parsing.py b/eeg_eyetracking_parser/_parsing.py
index c85cfec..3add817 100644
--- a/eeg_eyetracking_parser/_parsing.py
+++ b/eeg_eyetracking_parser/_parsing.py
@@ -258,7 +258,8 @@ def only_trial(name): return name == 'trial'
     if '' in dm.t_onset_1:
         valid_rows = dm[dm[dm.t_onset_1] != '']
         logger.warning(
-            f'ignoring {len(dm) - len(valid_rows)} rows of eye-tracking data without epoch 1')
+            f'ignoring {len(dm) - len(valid_rows)} rows of eye-tracking data '
+            f'without epoch 1')
         bigdm = bigdm[valid_rows]
         dm = dm[valid_rows]
     # Check for missing EEG triggers. If the first trigger is missing, a
@@ -266,8 +267,13 @@ def only_trial(name): return name == 'trial'
     # removed also from dm and bigdm.
     triggers = trial_trigger(events)[:, 2]
     if triggers[0] != 128:
-        raise ValueError(
-            f'The first trial trigger is {triggers[0]}, should be 128')
+        n_missing_trials = triggers[0] - 128
+        logger.warning(
+            f'The first trial trigger is {triggers[0]}, should be 128. '
+            f'Skipping {n_missing_trials} first trials of eye-tracking data '
+            f'to maintain alignment.')
+        dm = dm[n_missing_trials:]
+        bigdm = bigdm[n_missing_trials:]
     rows = list(range(len(dm)))
     for i, (tr1, tr2) in enumerate(zip(triggers[:-1], triggers[1:])):
         if tr2 - tr1 not in (1, -127):
@@ -282,7 +288,9 @@ def only_trial(name): return name == 'trial'
     missing = len(dm) - len(triggers)
     if missing > 0:
         logger.warning(
-            f'final {missing} triggers missing from recording, truncating eye data with one extra trial because the last trial may be incomplete too')
+            f'final {missing} triggers missing from recording, truncating eye '
+            f'data with one extra trial because the last trial may be '
+            f'incomplete too')
         dm = dm[:-missing - 1]
         bigdm = bigdm[:-missing - 1]
         last_trigger_index = np.where(events[0][:, 2] >= 128)[0][-1]
@@ -421,7 +429,7 @@ def _read_eeg_data(eeg_path, trigger_parser, margin):
         logger.info(f'trimming eeg to 0 - {end} s')
         raw.crop(0, end)
     logger.info('validating events')
-    _validate_events(events)
+    events = _validate_events(events[0]), events[1]
     logger.info('creating annotations from events')
     raw.set_annotations(
         mne.annotations_from_events(
diff --git a/eeg_eyetracking_parser/_triggers.py b/eeg_eyetracking_parser/_triggers.py
index a7ad7fa..8b1e8a3 100644
--- a/eeg_eyetracking_parser/_triggers.py
+++ b/eeg_eyetracking_parser/_triggers.py
@@ -70,7 +70,8 @@ def _validate_events(events):
     for i in range(len(events) - 1):
         dt = events[i + 1, 0] - events[i, 0]
         if dt == 1:
-            logger.warning(f'ignoring ghost trigger: {events[i]}')
+            logger.warning(
+                f'ignoring ghost trigger: {events[i]} before {events[i + 1]}')
             continue
         valid_events.append(events[i])
     events = np.array(valid_events)
@@ -81,13 +82,27 @@ def _validate_events(events):
         raise ValueError('trigger codes should be values between 1 and 255')
     # Check for duplicate triggers within trials
     trialid = -1
-    triggers = []
+    triggers_in_trial = []
+    select_triggers = []
     for code in codes:
+        # Detect trial triggers and always keep them
         if code >= 128:
             trialid += 1
-            triggers = []
-        if code in triggers:
-            raise ValueError(
+            triggers_in_trial = []
+            select_triggers.append(True)
+            continue
+        # Skip triggers that precede a trial onset
+        if trialid == -1:
+            logger.warning(f'trigger {code} precedes first trial')
+            select_triggers.append(False)
+            continue
+        # Skip duplicate triggers within a trial
+        if code in triggers_in_trial:
+            logger.warning(
                 f'duplicate trigger {code} in trial {trialid}, label {hex(255 - 128 - trialid % 128)}')
-        triggers.append(code)
-    return events       
+            select_triggers.append(False)
+        else:
+            select_triggers.append(True)
+        triggers_in_trial.append(code)
+    events = events[select_triggers]
+    return events