Merge pull request #60 from bukosabino/develop-fix-schedules2

Fix schedule jobs to use just one initialization
bukosabino · Feb 5, 2024 · e4cafa5 · e4cafa5
2 parents 12eafc8 + ef47d97
commit e4cafa5
Show file tree

Hide file tree

Showing 10 changed files with 53 additions and 89 deletions.
diff --git a/src/etls/bocm/daily_job.py b/src/etls/bocm/daily_job.py
diff --git a/src/etls/bocm/load.py b/src/etls/bocm/load.py
@@ -5,17 +5,18 @@
 from src.email.send_email import send_email
 from src.etls.bocm.scrapper import BOCMScrapper
 from src.etls.common.etl import ETL
-from src.initialize import initialize_app
 from src.etls.bocm.defs import COLLECTION_NAME
+from src.initialize import initialize_app
 
 
 app = typer.Typer()
-INIT_OBJECTS = initialize_app()
 
 
 @app.command()
-def today():
-    etl_job = ETL(config_loader=INIT_OBJECTS.config_loader, vector_store=INIT_OBJECTS.vector_store[COLLECTION_NAME])
+def today(init_objects=None):
+    if init_objects is None:
+        init_objects = initialize_app()
+    etl_job = ETL(config_loader=init_objects.config_loader, vector_store=init_objects.vector_store[COLLECTION_NAME])
     bocm_scrapper = BOCMScrapper()
     day = date.today()
     docs = bocm_scrapper.download_day(day)
@@ -27,14 +28,16 @@ def today():
     Daily ETL executed
     - Date: {day}
     - Documents loaded: {len(docs)} 
-    - Database used: {INIT_OBJECTS.config_loader['vector_store']}
+    - Database used: {init_objects.config_loader['vector_store']}
     """
-    send_email(INIT_OBJECTS.config_loader, subject, content)
+    send_email(init_objects.config_loader, subject, content)
 
 
 @app.command()
-def dates(date_start: str, date_end: str):
-    etl_job = ETL(config_loader=INIT_OBJECTS.config_loader, vector_store=INIT_OBJECTS.vector_store[COLLECTION_NAME])
+def dates(date_start: str, date_end: str, init_objects=None):
+    if init_objects is None:
+        init_objects = initialize_app()
+    etl_job = ETL(config_loader=init_objects.config_loader, vector_store=init_objects.vector_store[COLLECTION_NAME])
     bocm_scrapper = BOCMScrapper()
     docs = bocm_scrapper.download_days(
         date_start=datetime.strptime(date_start, "%Y/%m/%d").date(),
@@ -49,9 +52,9 @@ def dates(date_start: str, date_end: str):
     - Date start: {date_start}
     - Date end: {date_end}
     - Documents loaded: {len(docs)} 
-    - Database used: {INIT_OBJECTS.config_loader['vector_store']}
+    - Database used: {init_objects.config_loader['vector_store']}
     """
-    send_email(INIT_OBJECTS.config_loader, subject, content)
+    send_email(init_objects.config_loader, subject, content)
 
 
 if __name__ == "__main__":

diff --git a/src/etls/boe/daily_job.py b/src/etls/boe/daily_job.py
diff --git a/src/etls/boe/load.py b/src/etls/boe/load.py
@@ -5,17 +5,18 @@
 from src.email.send_email import send_email
 from src.etls.boe.scrapper import BOEScrapper
 from src.etls.common.etl import ETL
-from src.initialize import initialize_app
 from src.etls.boe.defs import COLLECTION_NAME
+from src.initialize import initialize_app
 
 
 app = typer.Typer()
-INIT_OBJECTS = initialize_app()
 
 
 @app.command()
-def today():
-    etl_job = ETL(config_loader=INIT_OBJECTS.config_loader, vector_store=INIT_OBJECTS.vector_store[COLLECTION_NAME])
+def today(init_objects=None):
+    if init_objects is None:
+        init_objects = initialize_app()
+    etl_job = ETL(config_loader=init_objects.config_loader, vector_store=init_objects.vector_store[COLLECTION_NAME])
     boe_scrapper = BOEScrapper()
     day = date.today()
     docs = boe_scrapper.download_day(day)
@@ -27,14 +28,16 @@ def today():
     Daily ETL executed
     - Date: {day}
     - Documents loaded: {len(docs)} 
-    - Database used: {INIT_OBJECTS.config_loader['vector_store']}
+    - Database used: {init_objects.config_loader['vector_store']}
     """
-    send_email(INIT_OBJECTS.config_loader, subject, content)
+    send_email(init_objects.config_loader, subject, content)
 
 
 @app.command()
-def dates(date_start: str, date_end: str):
-    etl_job = ETL(config_loader=INIT_OBJECTS.config_loader, vector_store=INIT_OBJECTS.vector_store[COLLECTION_NAME])
+def dates(date_start: str, date_end: str, init_objects=None):
+    if init_objects is None:
+        init_objects = initialize_app()
+    etl_job = ETL(config_loader=init_objects.config_loader, vector_store=init_objects.vector_store[COLLECTION_NAME])
     boe_scrapper = BOEScrapper()
     docs = boe_scrapper.download_days(
         date_start=datetime.strptime(date_start, "%Y/%m/%d").date(),
@@ -49,9 +52,9 @@ def dates(date_start: str, date_end: str):
     - Date start: {date_start}
     - Date end: {date_end}
     - Documents loaded: {len(docs)} 
-    - Database used: {INIT_OBJECTS.config_loader['vector_store']}
+    - Database used: {init_objects.config_loader['vector_store']}
     """
-    send_email(INIT_OBJECTS.config_loader, subject, content)
+    send_email(init_objects.config_loader, subject, content)
 
 
 if __name__ == "__main__":

diff --git a/src/etls/bopz/daily_job.py b/src/etls/bopz/daily_job.py
diff --git a/src/etls/bopz/load.py b/src/etls/bopz/load.py
@@ -10,12 +10,13 @@
 
 
 app = typer.Typer()
-INIT_OBJECTS = initialize_app()
 
 
 @app.command()
-def today():
-    etl_job = ETL(config_loader=INIT_OBJECTS.config_loader, vector_store=INIT_OBJECTS.vector_store[COLLECTION_NAME])
+def today(init_objects=None):
+    if init_objects is None:
+        init_objects = initialize_app()
+    etl_job = ETL(config_loader=init_objects.config_loader, vector_store=init_objects.vector_store[COLLECTION_NAME])
     bopz_scrapper = BOPZScrapper()
     day = date.today()
     docs = bopz_scrapper.download_day(day)
@@ -27,14 +28,16 @@ def today():
     Daily ETL executed
     - Date: {day}
     - Documents loaded: {len(docs)} 
-    - Database used: {INIT_OBJECTS.config_loader['vector_store']}
+    - Database used: {init_objects.config_loader['vector_store']}
     """
-    send_email(INIT_OBJECTS.config_loader, subject, content)
+    send_email(init_objects.config_loader, subject, content)
 
 
 @app.command()
-def dates(date_start: str, date_end: str):
-    etl_job = ETL(config_loader=INIT_OBJECTS.config_loader, vector_store=INIT_OBJECTS.vector_store[COLLECTION_NAME])
+def dates(date_start: str, date_end: str, init_objects=None):
+    if init_objects is None:
+        init_objects = initialize_app()
+    etl_job = ETL(config_loader=init_objects.config_loader, vector_store=init_objects.vector_store[COLLECTION_NAME])
     bopz_scrapper = BOPZScrapper()
     docs = bopz_scrapper.download_days(
         date_start=datetime.strptime(date_start, "%Y/%m/%d").date(),
@@ -49,9 +52,9 @@ def dates(date_start: str, date_end: str):
     - Date start: {date_start}
     - Date end: {date_end}
     - Documents loaded: {len(docs)} 
-    - Database used: {INIT_OBJECTS.config_loader['vector_store']}
+    - Database used: {init_objects.config_loader['vector_store']}
     """
-    send_email(INIT_OBJECTS.config_loader, subject, content)
+    send_email(init_objects.config_loader, subject, content)
 
 
 if __name__ == "__main__":

diff --git a/src/etls/template/README.md b/src/etls/template/README.md
@@ -18,7 +18,7 @@ To define an ETL for your gazette, you need to fill some files:
 If you want to do a batch/historical load:
 
 ```sh
-python -m src.etls.template.load dates <collection_name> <date_start_%Y/%m/%d> <date_end_%Y/%m/%d>
+python -m src.etls.template.load dates <date_start_%Y/%m/%d> <date_end_%Y/%m/%d>
 ```
 
 Note: You should update the end/start dates in the `config/config.py' file.
@@ -28,7 +28,7 @@ Note: You should update the end/start dates in the `config/config.py' file.
 Most likely, your Gazette will be updated every day, so you will need to run a daily ETL script. Take a look at src.etls.template.load.py for inspiration.
 
 ```sh
-python -m src.etls.template.load today <collection_name>
+python -m src.etls.template.load today
 ```
 
 You will probably also want to schedule a daily job to update your embedding database. Then take a look at `src/etls/template/schedule.py`.

diff --git a/src/etls/template/daily_job.py b/src/etls/template/daily_job.py
diff --git a/src/etls/template/load.py b/src/etls/template/load.py
@@ -5,17 +5,18 @@
 from src.email.send_email import send_email
 from src.etls.template.scrapper import TemplateScrapper
 from src.etls.common.etl import ETL
-from src.initialize import initialize_app
 from src.etls.template.defs import COLLECTION_NAME
+from src.initialize import initialize_app
 
 
-INIT_OBJECTS = initialize_app()
 app = typer.Typer()
 
 
 @app.command()
-def today():
-    etl_job = ETL(config_loader=INIT_OBJECTS.config_loader, vector_store=INIT_OBJECTS.vector_store[COLLECTION_NAME])
+def today(init_objects=None):
+    if init_objects is None:
+        init_objects = initialize_app()
+    etl_job = ETL(config_loader=init_objects.config_loader, vector_store=init_objects.vector_store[COLLECTION_NAME])
     boe_scrapper = TemplateScrapper()
     day = date.today()
     docs = boe_scrapper.download_day(day)
@@ -27,14 +28,16 @@ def today():
         Today ETL executed
         - Date: {day}
         - Documents loaded: {len(docs)} 
-        - Database used: {INIT_OBJECTS.config_loader['vector_store']}
+        - Database used: {init_objects.config_loader['vector_store']}
         """
-    send_email(INIT_OBJECTS.config_loader, subject, content)
+    send_email(init_objects.config_loader, subject, content)
 
 
 @app.command()
-def dates(date_start: str, date_end: str):
-    etl_job = ETL(config_loader=INIT_OBJECTS.config_loader, vector_store=INIT_OBJECTS.vector_store[COLLECTION_NAME])
+def dates(date_start: str, date_end: str, init_objects=None):
+    if init_objects is None:
+        init_objects = initialize_app()
+    etl_job = ETL(config_loader=init_objects.config_loader, vector_store=init_objects.vector_store[COLLECTION_NAME])
     scrapper = TemplateScrapper()
     docs = scrapper.download_days(
         date_start=datetime.strptime(date_start, "%Y/%m/%d").date(),
@@ -49,9 +52,9 @@ def dates(date_start: str, date_end: str):
     - Date start: {date_start}
     - Date end: {date_end}
     - Documents loaded: {len(docs)} 
-    - Database used: {INIT_OBJECTS.config_loader['vector_store']}
+    - Database used: {init_objects.config_loader['vector_store']}
     """
-    send_email(INIT_OBJECTS.config_loader, subject, content)
+    send_email(init_objects.config_loader, subject, content)
 
 
 if __name__ == "__main__":

diff --git a/src/etls/template/monthly_job.py b/src/etls/template/monthly_job.py