Merge pull request #66 from beeldengeluid/various-fixes

various fixes

.github/workflows/_test.yml

@@ -14,7 +14,7 @@ jobs:
     - name: "Install Python"
       uses: "actions/setup-python@v5"
       with:
-        python-version: "3.10"
+        python-version: "3.11"
 
     - name: "Install poetry"
       run: "pip install --user poetry"

Dockerfile

@@ -5,7 +5,6 @@ RUN apt-get clean && apt-get update -y && apt-get upgrade -y
 RUN apt-get install -y libgl1 ffmpeg
 
 RUN pip install --upgrade pip
-RUN pip install scenedetect[opencv] --upgrade
 RUN pip install poetry==1.5.1
 
 # Create dirs for:

docker-entrypoint.sh

@@ -3,7 +3,3 @@
 echo "Starting DANE video segmentation worker"
 
 poetry run python worker.py "$@"
-
-echo The worker crashed, tailing /dev/null for debugging
-
-tail -f /dev/null

io_util.py

@@ -201,18 +201,6 @@ def delete_input_file(input_file: str, actually_delete: bool) -> bool:
         logger.exception("Could not delete input file")
         return False
 
-    # now remove the "chunked path" from /mnt/dane-fs/input-files/03/d2/8a/03d28a03643a981284b403b91b95f6048576c234/xyz.mp4
-    try:
-        os.chdir(get_download_dir())  # cd /mnt/dane-fs/input-files
-        os.removedirs(
-            f".{input_file[len(get_download_dir()):input_file.rfind(os.sep)]}"
-        )  # /03/d2/8a/03d28a03643a981284b403b91b95f6048576c234
-        logger.info("Deleted empty input dirs too")
-    except OSError:
-        logger.exception("OSError while removing empty input file dirs")
-    except FileNotFoundError:
-        logger.exception("FileNotFoundError while removing empty input file dirs")
-
     return True  # return True even if empty dirs were not removed
 
 

pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Jaap Blom <jblom@beeldengeluid.nl>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.11"
 yacs = "^0.1.8"
 pika = "^1.3.2"
 requests = "^2.32.0"
@@ -42,7 +42,7 @@ omit = [
 
 [tool.black]
 line-length = 88
-target_version = ['py310']
+target_version = ['py311']
 include = '\.pyi?$'
 exclude = '''
 
@@ -58,7 +58,7 @@ exclude = '''
 '''
 
 [tool.mypy]
-python_version = "3.10"
+python_version = "3.11"
 
 [[tool.mypy.overrides]]
 module = [

worker.py

@@ -79,12 +79,15 @@ def callback(self, task: Task, doc: Document) -> CallbackResponse:
             logger.info(
                 "applying IO on output went well, now finally saving to DANE index"
             )
-            self.save_to_dane_index(
-                doc,
-                task,
-                get_s3_output_file_uri(get_source_id(input_file_path)),
-                provenance=full_provenance_chain,
-            )
+            try:
+                self.save_to_dane_index(
+                    doc,
+                    task,
+                    get_s3_output_file_uri(get_source_id(input_file_path)),
+                    provenance=full_provenance_chain,
+                )
+            except Exception as e:
+                logger.exception(f"Failed to save to DANE index: {e}")
         return processing_result
 
     def save_to_dane_index(