From 576b1cf5a31e884adacf201b78db94eee8ae538d Mon Sep 17 00:00:00 2001 From: stevensotelo Date: Wed, 13 Sep 2023 09:36:50 -0500 Subject: [PATCH] in complete data filter the file issue.csv and we won't process it. also include a log in prediccionClimatica folder to check what is on going --- .gitignore | 6 ++++++ src/aclimate_resampling/complete_data.py | 14 +++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 98a6a27..0ae819e 100644 --- a/.gitignore +++ b/.gitignore @@ -171,7 +171,13 @@ src/unittests.py /cdsapirc /.cdsapirc + +# TEST FILES + +data/local test/test_files +/test/complete_local_test.py +test/complete_local_test.py data/chirp.2023.06.01.tif data/Temperature-Air-2m-Max-24h_C3S-glob-agric_AgERA5_20230601_final-v1.0.tif diff --git a/src/aclimate_resampling/complete_data.py b/src/aclimate_resampling/complete_data.py index 126aed3..07a42ef 100644 --- a/src/aclimate_resampling/complete_data.py +++ b/src/aclimate_resampling/complete_data.py @@ -301,10 +301,8 @@ def extract_values(self,dir_path,var,locations, date_start,date_end,date_format) def filter_extract_data(self, data_frame): current_year = self.start_date.year current_month = self.start_date.month - - filter_data_frame = data_frame.loc[ - (data_frame["year"] <= current_year) & (data_frame["month"] <= current_month), - :] + + filter_data_frame = data_frame.loc[(data_frame["year"] <= current_year) & (data_frame["month"] <= current_month),:] return filter_data_frame @@ -466,8 +464,10 @@ def run(self): print("Listing stations") df_ws = self.list_ws() - df_ws_full = df_ws[df_ws['message'].isna()] - df_ws_nan = df_ws[~df_ws['message'].isna()] + df_ws_full = df_ws.loc[df_ws['message'] == "",:] + df_ws_nan = df_ws.loc[df_ws['message'] != "",:] + df_ws_full.to_csv(os.path.join(self.path_country_outputs_forecast,"resampling_complete_data_stations_coord.csv"),index=False) + df_ws_nan.to_csv(os.path.join(self.path_country_outputs_forecast,"resampling_complete_data_stations_without_coord.csv"),index=False) print("Listed stations") print("Adding data started!") pool = mp.Pool(processes=self.cores) @@ -477,5 +477,5 @@ def run(self): process.get() print("Added data!") - df_ws_nan.to_csv(os.path.join(self.path_country_outputs_forecast,"stations_without_coord.csv"),index=False) + print("Process finished")