Handling exceptions for scheduled jobs

src/etls/boa/load.py

@@ -5,13 +5,15 @@
 
 from src.email.send_email import send_email
 from src.etls.boa.scrapper import BOAScrapper
+from src.etls.utils import catch_exceptions
 from src.etls.boa.defs import COLLECTION_NAME
 from src.etls.common.etl import ETL
 from src.initialize import initialize_app
 
 app = typer.Typer()
 
 @app.command()
+@catch_exceptions(cancel_on_failure=True)
 def today(init_objects=None):
     if init_objects is None:
         init_objects = initialize_app()

src/etls/bocm/load.py

@@ -5,6 +5,7 @@
 from src.email.send_email import send_email
 from src.etls.bocm.scrapper import BOCMScrapper
 from src.etls.common.etl import ETL
+from src.etls.utils import catch_exceptions
 from src.etls.bocm.defs import COLLECTION_NAME
 from src.initialize import initialize_app
 
@@ -13,6 +14,7 @@
 
 
 @app.command()
+@catch_exceptions(cancel_on_failure=True)
 def today(init_objects=None):
     if init_objects is None:
         init_objects = initialize_app()

src/etls/boe/load.py

@@ -5,6 +5,7 @@
 from src.email.send_email import send_email
 from src.etls.boe.scrapper import BOEScrapper
 from src.etls.common.etl import ETL
+from src.etls.utils import catch_exceptions
 from src.etls.boe.defs import COLLECTION_NAME
 from src.initialize import initialize_app
 
@@ -13,6 +14,7 @@
 
 
 @app.command()
+@catch_exceptions(cancel_on_failure=True)
 def today(init_objects=None):
     if init_objects is None:
         init_objects = initialize_app()

src/etls/boja/load.py

@@ -4,13 +4,15 @@
 
 from src.email.send_email import send_email
 from src.etls.boja.scrapper import BOJAScrapper
+from src.etls.utils import catch_exceptions
 from src.etls.boja.defs import COLLECTION_NAME
 from src.etls.common.etl import ETL
 from src.initialize import initialize_app
 
 app = typer.Typer()
 
 @app.command()
+@catch_exceptions(cancel_on_failure=True)
 def today(init_objects=None):
     if init_objects is None:
         init_objects = initialize_app()

src/etls/bopv/load.py

@@ -6,12 +6,14 @@
 from src.email.send_email import send_email
 from src.etls.bopv.scrapper import BOPVScrapper
 from src.etls.bopv.defs import COLLECTION_NAME
+from src.etls.utils import catch_exceptions
 from src.etls.common.etl import ETL
 from src.initialize import initialize_app
 
 app = typer.Typer()
 
 @app.command()
+@catch_exceptions(cancel_on_failure=True)
 def today(init_objects=None):
     if init_objects is None:
         init_objects = initialize_app()

src/etls/bopz/load.py

@@ -5,6 +5,7 @@
 from src.email.send_email import send_email
 from src.etls.bopz.scrapper import BOPZScrapper
 from src.etls.bopz.defs import COLLECTION_NAME
+from src.etls.utils import catch_exceptions
 from src.etls.common.etl import ETL
 from src.initialize import initialize_app
 
@@ -13,6 +14,7 @@
 
 
 @app.command()
+@catch_exceptions(cancel_on_failure=True)
 def today(init_objects=None):
     if init_objects is None:
         init_objects = initialize_app()

src/etls/utils.py

@@ -0,0 +1,16 @@
+import functools
+
+
+def catch_exceptions(cancel_on_failure=False):
+    def catch_exceptions_decorator(job_func):
+        @functools.wraps(job_func)
+        def wrapper(*args, **kwargs):
+            try:
+                return job_func(*args, **kwargs)
+            except:
+                import traceback
+                print(traceback.format_exc())
+                if cancel_on_failure:
+                    return schedule.CancelJob
+        return wrapper
+    return catch_exceptions_decorator