Merge pull request #52 from spaceml-org/7-patch-extension

7 patch extension (draft pull request)

config/example/datamodule.yaml

@@ -3,9 +3,9 @@ dataloader:
 
   datasets_spec:
     msg:
-      data_path: /path/to/your/data
+      data_path: /path/to/msg/data
     goes:
-      data_path: /path/to/your/data
+      data_path: /path/to/goes/data
   load_coords: True
   load_cloudmask: True
 

config/example/download.yaml

@@ -1,15 +1,15 @@
 # PERIOD
 period:
-  start_date: '2020-10-01'
+  start_date: '2020-01-01'
   start_time: '00:00:00'
-  end_date: '2020-10-31'
+  end_date: '2020-12-31'
   end_time: '23:59:00'
 
 # CLOUD MASK
 cloud_mask: True
 
 # PATH FOR SAVING DATA
-save_dir: data
+save_dir: /path/to/save/data
 
 defaults:
   - _self_

config/example/geoprocess.yaml

@@ -1,8 +1,8 @@
 # PATH WITH RAW DATA
-read_path: data
+read_path: /path/to/donwloaded/data
 
 # PATH FOR SAVING GEOPROCESSED DATA
-save_path: data
+save_path: /path/to/save/geoprocessed/data
 
 defaults:
   - _self_

config/example/main.yaml

@@ -1,7 +1,7 @@
 defaults:
   - download
   - geoprocess
-  - patch
-  - satellite: terra
+  # - patch
+  - satellite: msg
 
-stage: patch
+stage: geoprocess

config/example/patch.yaml

@@ -1,8 +1,8 @@
 # PATH WITH GEOPROCESSED DATA
-read_path: data
+read_path: /path/to/geoprocessed/data
 
 # PATH FOR SAVING PATCHES
-save_path: data
+save_path: /path/to/save/patches
 
 # PATCH PARAMETERS
 patch_size: 256
@@ -11,7 +11,7 @@ stride_size: 256
 # NAN CUTOFF
 nan_cutoff: 0.5
 
-# FILETYPE TO SAVE [nc = netcdf, np = numpy]
+# FILETYPE TO SAVE [nc = netcdf, tif = geotiff, np = numpy, npz = compressed numpy]
 save_filetype: nc
 
 defaults:

config/example/satellite/aqua.yaml

@@ -13,8 +13,6 @@ geoprocess:
   save_path: ${save_path}/aqua/geoprocessed
   satellite: aqua
 
-# preprocess:
-
 patch:
   _target_: rs_tools._src.preprocessing.prepatcher.prepatch
   read_path: ${read_path}/aqua/geoprocessed

config/example/satellite/goes.yaml

@@ -5,20 +5,18 @@ download:
   start_time: ${period.start_time}
   end_date: ${period.end_date}
   end_time: ${period.end_time}
-  daily_window_t0: "14:00:00"
-  daily_window_t1: "20:00:00"
+  daily_window_t0: "00:00:00"
+  daily_window_t1: "23:59:00"
   time_step: "1:00:00"
 
 geoprocess:
   _target_: rs_tools._src.geoprocessing.goes.geoprocessor_goes16.geoprocess
-  read_path: ${read_path}/goes16/raw
-  save_path: ${save_path}/goes16/geoprocessed
-  resolution: null
-  region: "-130 -15 -90 5"
+  read_path: ${read_path}/goes-data/2020/split_${split}
+  save_path: ${save_path}/goes-geoprocessed/goes
+  resolution: null # MSG resolution = 3000.40
+  region: null
   resample_method: bilinear
 
-# preprocess:
-
 patch:
   _target_: rs_tools._src.preprocessing.prepatcher.prepatch
   read_path: ${read_path}/goes16/geoprocessed

config/example/satellite/msg.yaml

@@ -5,20 +5,18 @@ download:
   start_time: ${period.start_time}
   end_date: ${period.end_date}
   end_time: ${period.end_time}
-  daily_window_t0: "9:00:00"
-  daily_window_t1: "17:00:00"
+  daily_window_t0: "00:00:00"
+  daily_window_t1: "23:59:00"
   time_step: "1:00:00"
 
 geoprocess:
   _target_: rs_tools._src.geoprocessing.msg.geoprocessor_msg.geoprocess
-  read_path: ${read_path}/msg/raw
-  save_path: ${save_path}/msg/geoprocessed
+  read_path: ${read_path}/msg-data/msg
+  save_path: ${save_path}/msg-geoprocessed/msg
   resolution: null
-  region: "-70 -15 20 5"
+  region: null
   resample_method: bilinear
 
-# preprocess:
-
 patch:
   _target_: rs_tools._src.preprocessing.prepatcher.prepatch
   read_path: ${read_path}/msg/geoprocessed

config/example/satellite/terra.yaml

@@ -13,8 +13,6 @@ geoprocess:
   save_path: ${save_path}/terra/geoprocessed
   satellite: terra
 
-# preprocess:
-
 patch:
   _target_: rs_tools._src.preprocessing.prepatcher.prepatch
   read_path: ${read_path}/terra/geoprocessed

environments/environment.yaml

@@ -38,6 +38,9 @@ dependencies:
       - typer
       - einops
       - cartopy
+      - google-cloud-storage
+      - gcsfs
+      - xrpatcher
       # formatting
       - rasterio
       - black

notebooks/dev/goes/1.6-additional-download.ipynb

@@ -0,0 +1,103 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import glob\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "path = '/mnt/disks/goes-data/2020/CM/'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_list_filenames(data_path: str=\"./\", ext: str=\"*\"):\n",
+    "    \"\"\"\n",
+    "    Loads a list of file names within a directory.\n",
+    "\n",
+    "    Args:\n",
+    "        data_path (str, optional): The directory path to search for files. Defaults to \"./\".\n",
+    "        ext (str, optional): The file extension to filter the search. Defaults to \"*\".\n",
+    "\n",
+    "    Returns:\n",
+    "        List[str]: A sorted list of file names matching the given extension within the directory.\n",
+    "    \"\"\"\n",
+    "    pattern = f\"*{ext}\"\n",
+    "    return sorted(glob.glob(os.path.join(data_path, \"**\", pattern), recursive=True))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "files = get_list_filenames(path, ext=\"nc\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "8733"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(files)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "rs_tools",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}