Merge pull request huggingface#181 from QasidSaleem/remove_import_Lis…

…tFilter remove ListFilter from the process_common_crawl_dump example
swiss-ai · May 10, 2024 · 9f5f7b0 · 9f5f7b0
2 parents 4d83342 + e41e40c
commit 9f5f7b0
Showing 1 changed file with 0 additions and 2 deletions.
diff --git a/examples/process_common_crawl_dump.py b/examples/process_common_crawl_dump.py
@@ -6,7 +6,6 @@
     GopherQualityFilter,
     GopherRepetitionFilter,
     LanguageFilter,
-    ListFilter,
     URLFilter,
 )
 from datatrove.pipeline.readers import WarcReader
@@ -39,7 +38,6 @@
         ),
         GopherRepetitionFilter(exclusion_writer=JsonlWriter(f"{MAIN_OUTPUT_PATH}/removed/repetitive/{DUMP}")),
         GopherQualityFilter(exclusion_writer=JsonlWriter(f"{MAIN_OUTPUT_PATH}/removed/quality/{DUMP}")),
-        ListFilter(exclusion_writer=JsonlWriter(f"{MAIN_OUTPUT_PATH}/removed/list/{DUMP}")),
         JsonlWriter(f"{MAIN_OUTPUT_PATH}/output/{DUMP}"),
     ],
     tasks=8000,