From 893cbd4938d977f064d9f3843cd1effe8dc1f95a Mon Sep 17 00:00:00 2001 From: Lars Falk-Petersen Date: Thu, 4 Jan 2024 10:18:31 +0100 Subject: [PATCH] Add more URL tests in test_app. Split out position_page. Make somewhat instance-aware. --- app/routes/collections_page.py | 211 ++---------------------------- app/routes/instances_page.py | 129 +++++++++++++++++++ app/routes/position_page.py | 227 +++++++++++++++++++++++++++++++++ app/routes/routes.py | 4 + app/test_app.py | 35 ++++- 5 files changed, 405 insertions(+), 201 deletions(-) create mode 100644 app/routes/instances_page.py create mode 100644 app/routes/position_page.py diff --git a/app/routes/collections_page.py b/app/routes/collections_page.py index d203920..180ef0f 100644 --- a/app/routes/collections_page.py +++ b/app/routes/collections_page.py @@ -1,16 +1,10 @@ """Collections page.""" from functools import lru_cache -from typing import List from datetime import timedelta import logging -from fastapi import APIRouter, status, Response +from fastapi import APIRouter import edr_pydantic from edr_pydantic.collections import Collection -from pydantic import AwareDatetime -from shapely import wkt, GEOSException, Point -import covjson_pydantic -from covjson_pydantic.coverage import Coverage -from covjson_pydantic.ndarray import NdArray from initialize import get_dataset, BASE_URL @@ -18,12 +12,6 @@ get_vertical_extent, get_spatial_extent, get_temporal_extent, - TEMPERATURE_LABEL, - LAT_LABEL, - LON_LABEL, - UWIND_LABEL, - VWIND_LABEL, - ISOBARIC_LABEL, ) router = APIRouter() @@ -31,8 +19,11 @@ @lru_cache -def create_collection(collection_id: str = "") -> dict: +def create_collection(collection_id: str = "", instance: str = "") -> dict: """Creates the collections page.""" + + # TODO: Check for instance + link_self = edr_pydantic.link.Link( href=BASE_URL, hreflang="en", rel="self", type="aplication/json" ) @@ -98,7 +89,7 @@ def create_collection(collection_id: str = "") -> dict: href=f"{collection_url}/instances", rel="data", variables=edr_pydantic.variables.Variables( - query_type="instance", output_formats=["CoverageJSON"] + query_type="instances", output_formats=["CoverageJSON"] ), ) ), @@ -151,180 +142,6 @@ def create_collection(collection_id: str = "") -> dict: return isobaric_col.model_dump(exclude_none=True) -def create_point(coords: str = "") -> dict: - """Return data for all isometric layers at a point.""" - point = Point() - try: - point = wkt.loads(coords) - except GEOSException: - errmsg = ( - "Error, coords should be a Well Known Text, for example " - + f'"POINT(11.0 59.0)". You gave "{coords}"' - ) - logger.error(errmsg) - return Response(status_code=status.HTTP_400_BAD_REQUEST, content=errmsg) - - logger.info("create_data for coord %s, %s", point.y, point.x) - dataset = get_dataset() - - # Sanity checks on coordinates - if ( - point.y > dataset[TEMPERATURE_LABEL][LAT_LABEL].values.max() - or point.y < dataset[TEMPERATURE_LABEL][LAT_LABEL].values.min() - ): - errmsg = ( - f"Error, coord {point.y} out of bounds. Min/max is " - + "{dataset[TEMPERATURE_LABEL][LAT_LABEL].values.min()}/" - + "{dataset[TEMPERATURE_LABEL][LAT_LABEL].values.max()}" - ) - logger.error(errmsg) - return Response(status_code=status.HTTP_400_BAD_REQUEST, content=errmsg) - if ( - point.x > dataset[TEMPERATURE_LABEL][LON_LABEL].values.max() - or point.x < dataset[TEMPERATURE_LABEL][LON_LABEL].values.min() - ): - errmsg = f"Error, coord {point.x} out of bounds. Min/max is \ - {dataset[TEMPERATURE_LABEL][LON_LABEL].values.min()}/\ - {dataset[TEMPERATURE_LABEL][LON_LABEL].values.max()}" - logger.error(errmsg) - return Response(status_code=status.HTTP_400_BAD_REQUEST, content=errmsg) - - # Fetch temperature - temperatures = dataset[TEMPERATURE_LABEL].sel( - longitude=point.x, latitude=point.y, method="nearest" - ) - - isobaric_values = get_vertical_extent(dataset) - temperature_values: List[float | None] = [] - uwind_values: List[float | None] = [] - vwind_values: List[float | None] = [] - - for temperature in temperatures: - temperature_values.append(float(temperature.data)) - - uwind = dataset[UWIND_LABEL].sel( - longitude=point.x, - latitude=point.y, - isobaricInhPa=temperature[ISOBARIC_LABEL].data, - method="nearest", - ) - uwind_values.append(float(uwind.data)) - - vwind = dataset[VWIND_LABEL].sel( - longitude=point.x, - latitude=point.y, - isobaricInhPa=temperature[ISOBARIC_LABEL].data, - method="nearest", - ) - vwind_values.append(float(vwind.data)) - - cov = Coverage( - id="isobaric", - type="Coverage", - domain=covjson_pydantic.domain.Domain( - domainType=covjson_pydantic.domain.DomainType.vertical_profile, - axes=covjson_pydantic.domain.Axes( - x=covjson_pydantic.domain.ValuesAxis[float](values=[point.x]), - y=covjson_pydantic.domain.ValuesAxis[float](values=[point.y]), - z=covjson_pydantic.domain.ValuesAxis[float](values=isobaric_values), - t=covjson_pydantic.domain.ValuesAxis[AwareDatetime]( - values=[get_temporal_extent(dataset)] - ), - ), - referencing=[ - covjson_pydantic.reference_system.ReferenceSystemConnectionObject( - coordinates=["x", "y"], - system=covjson_pydantic.reference_system.ReferenceSystem( - id="http://www.opengis.net/def/crs/OGC/1.3/CRS84", - type="GeographicCRS", - ), - ), - covjson_pydantic.reference_system.ReferenceSystemConnectionObject( - coordinates=["z"], - system=covjson_pydantic.reference_system.ReferenceSystem( - type="VerticalCRS", - cs={ - "csAxes": [ - { - "name": {"en": "Pressure"}, - "direction": "down", - "unit": {"symbol": "Pa"}, - } - ] - }, - ), - ), - covjson_pydantic.reference_system.ReferenceSystemConnectionObject( - coordinates=["t"], - system=covjson_pydantic.reference_system.ReferenceSystem( - type="TemporalRS", calendar="Gregorian" - ), - ), - ], - ), - ranges={ - "temperature": NdArray( - axisNames=["z"], - shape=[len(isobaric_values)], - values=temperature_values, - ), - "uwind": NdArray( - axisNames=["z"], - shape=[len(isobaric_values)], - values=uwind_values, - ), - "vwind": NdArray( - axisNames=["z"], - shape=[len(isobaric_values)], - values=vwind_values, - ), - }, - parameters={ - "temperature": covjson_pydantic.parameter.Parameter( - id="temperature", - label={"en": "Air temperature"}, - observedProperty=covjson_pydantic.observed_property.ObservedProperty( - id="https://codes.wmo.int/common/quantity-kind/_airTemperature", - label={"en": "Air temperature"}, - ), - unit=covjson_pydantic.unit.Unit( - id="https://codes.wmo.int/common/unit/_K", - label={"en": "Kelvin"}, - symbol="K", - ), - ), - "uwind": covjson_pydantic.parameter.Parameter( - id="uwind", - label={"en": "U component of wind"}, - observedProperty=covjson_pydantic.observed_property.ObservedProperty( - id="https://codes.wmo.int/bufr4/b/11/_095", - label={"en": "u-component of wind"}, - ), - unit=covjson_pydantic.unit.Unit( - id="https://codes.wmo.int/common/unit/_m_s-1", - label={"en": "m/s"}, - symbol="m/s", - ), - ), - "vwind": covjson_pydantic.parameter.Parameter( - id="vwind", - label={"en": "V component of wind"}, - observedProperty=covjson_pydantic.observed_property.ObservedProperty( - id="https://codes.wmo.int/bufr4/b/11/_096", - label={"en": "v-component of wind"}, - ), - unit=covjson_pydantic.unit.Unit( - id="https://codes.wmo.int/common/unit/_m_s-1", - label={"en": "m/s"}, - symbol="m/s", - ), - ), - }, - ) - - return cov.model_dump(exclude_none=True) - - @router.get("/collections") async def create_collections_page() -> dict: """List all collections available. @@ -337,17 +154,13 @@ async def create_collections_page() -> dict: return create_collection() -@router.get("/collections/isobaric/position") -async def create_isobaric_page(coords: str) -> dict: - """Position. - - This is the main function of this API. Needs a string with the coordinates, and will return data for the nearest point. - - """ - return create_point(coords=coords) - - @router.get("/collections/{collection_id}") async def create_collection_page(collection_id: str) -> dict: """Show a specific collection. Isobaric is the only one available. No data is returned, only info about the collection.""" return create_collection(collection_id) + + +@router.get("/collections/{collection_id}/instances/{instance}/") +async def create_instance_collection_page(collection_id: str, instance: str) -> dict: + """Show a specific instance of a collection. Isobaric is the only one available, and the date in current file is the only instance available. No data is returned, only info about the collection.""" + return create_collection(collection_id, instance) diff --git a/app/routes/instances_page.py b/app/routes/instances_page.py new file mode 100644 index 0000000..c70201c --- /dev/null +++ b/app/routes/instances_page.py @@ -0,0 +1,129 @@ +"""Collections page.""" +from datetime import timedelta +import logging +from fastapi import APIRouter +import edr_pydantic +from edr_pydantic.collections import Instances, Instance + +from initialize import get_dataset, BASE_URL + +from grib import ( + get_vertical_extent, + get_spatial_extent, + get_temporal_extent, +) + +router = APIRouter() +logger = logging.getLogger() + + +def create_instances() -> dict: + """List all instances (dates) available in data file.""" + dataset = get_dataset() + vertical_levels = get_vertical_extent(dataset) + collection_url = f"{BASE_URL}collections/isobaric" + instance_dates = [get_temporal_extent(dataset)] + + instance_list = [] + for d in instance_dates: + formatted_date = d.strftime("%Y%m%d%H0000") + instance_list.append( + Instance( + id=formatted_date, + title=formatted_date, + description=f"Data from {formatted_date}", + links=[ + edr_pydantic.link.Link( + href=f"{collection_url}/instances/", + hreflang="en", + rel="self", + type="aplication/json", + ), + ], + extent=edr_pydantic.extent.Extent( + spatial=edr_pydantic.extent.Spatial( + bbox=[get_spatial_extent(dataset)], crs="WGS84" + ), + vertical=edr_pydantic.extent.Vertical( + interval=[ + [vertical_levels[0]], + [vertical_levels[len(vertical_levels) - 1]], + ], + values=vertical_levels, + vrs="Vertical Reference System: PressureLevel", # opendata.fmi.fi + ), + temporal=edr_pydantic.extent.Temporal( + interval=[ + [ + get_temporal_extent(dataset), + get_temporal_extent(dataset) + timedelta(hours=12), + ] + ], + values=[get_temporal_extent(dataset).isoformat()], + trs='TIMECRS["DateTime",TDATUM["Gregorian Calendar"],' + + 'CS[TemporalDateTime,1],AXIS["Time (T)",future]', # opendata.fmi.fi + ), + ), + data_queries=edr_pydantic.data_queries.DataQueries( + # Get posision in instance + position=edr_pydantic.data_queries.EDRQuery( + link=edr_pydantic.data_queries.EDRQueryLink( + href=f"{collection_url}/instances/{formatted_date}/position", + rel="data", + variables=edr_pydantic.variables.Variables( + query_type="position", + output_formats=["CoverageJSON"], + # coords="Well Known Text POINT value i.e. POINT(10.9 60.1)", + ), + ) + ), + ), + parameter_names=edr_pydantic.parameter.Parameters( + { + "WindUMS": edr_pydantic.parameter.Parameter( + observedProperty=edr_pydantic.observed_property.ObservedProperty( + label="WindUMS" + ) + ), + "WindVMS": edr_pydantic.parameter.Parameter( + observedProperty=edr_pydantic.observed_property.ObservedProperty( + label="WindVMS" + ) + ), + "Air temperature": edr_pydantic.parameter.Parameter( + id="Temperature", + unit=edr_pydantic.unit.Unit( + symbol=edr_pydantic.unit.Symbol( + value="K", + type="https://codes.wmo.int/common/unit/_K", + ) + ), + observedProperty=edr_pydantic.observed_property.ObservedProperty( + id="https://codes.wmo.int/common/quantity-kind/_airTemperature", + label="Kelvin", + ), + ), + } + ), + ) + ) + + isobaric_inst = Instances( + links=[ + edr_pydantic.link.Link( + href=f"{collection_url}/instances/", + hreflang="en", + rel="self", + type="aplication/json", + ) + ], + instances=instance_list, + ) + + return isobaric_inst.model_dump(exclude_none=True) + + +@router.get("/collections/isobaric/instances") +async def create_isobaric_instances_page() -> dict: + """List available instances.""" + return create_instances() diff --git a/app/routes/position_page.py b/app/routes/position_page.py new file mode 100644 index 0000000..25da66d --- /dev/null +++ b/app/routes/position_page.py @@ -0,0 +1,227 @@ +"""Collections page.""" +from typing import List +import logging +from fastapi import APIRouter, status, Response, Request +from pydantic import AwareDatetime +from shapely import wkt, GEOSException, Point +import covjson_pydantic +from covjson_pydantic.coverage import Coverage +from covjson_pydantic.ndarray import NdArray + +from initialize import get_dataset + +from grib import ( + get_vertical_extent, + get_temporal_extent, + TEMPERATURE_LABEL, + LAT_LABEL, + LON_LABEL, + UWIND_LABEL, + VWIND_LABEL, + ISOBARIC_LABEL, +) + +router = APIRouter() +logger = logging.getLogger() + + +def create_point(coords: str = "", instance: str = "") -> dict: + """Return data for all isometric layers at a point.""" + # TODO: check instance + point = Point() + try: + point = wkt.loads(coords) + except GEOSException: + errmsg = ( + "Error, coords should be a Well Known Text, for example " + + f'"POINT(11.0 59.0)". You gave "{coords}"' + ) + logger.error(errmsg) + return Response(status_code=status.HTTP_400_BAD_REQUEST, content=errmsg) + + logger.info("create_data for coord %s, %s", point.y, point.x) + dataset = get_dataset() + + # Sanity checks on coordinates + if ( + point.y > dataset[TEMPERATURE_LABEL][LAT_LABEL].values.max() + or point.y < dataset[TEMPERATURE_LABEL][LAT_LABEL].values.min() + ): + errmsg = ( + f"Error, coord {point.y} out of bounds. Min/max is " + + "{dataset[TEMPERATURE_LABEL][LAT_LABEL].values.min()}/" + + "{dataset[TEMPERATURE_LABEL][LAT_LABEL].values.max()}" + ) + logger.error(errmsg) + return Response(status_code=status.HTTP_400_BAD_REQUEST, content=errmsg) + if ( + point.x > dataset[TEMPERATURE_LABEL][LON_LABEL].values.max() + or point.x < dataset[TEMPERATURE_LABEL][LON_LABEL].values.min() + ): + errmsg = f"Error, coord {point.x} out of bounds. Min/max is \ + {dataset[TEMPERATURE_LABEL][LON_LABEL].values.min()}/\ + {dataset[TEMPERATURE_LABEL][LON_LABEL].values.max()}" + logger.error(errmsg) + return Response(status_code=status.HTTP_400_BAD_REQUEST, content=errmsg) + + # Fetch temperature + temperatures = dataset[TEMPERATURE_LABEL].sel( + longitude=point.x, latitude=point.y, method="nearest" + ) + + isobaric_values = get_vertical_extent(dataset) + temperature_values: List[float | None] = [] + uwind_values: List[float | None] = [] + vwind_values: List[float | None] = [] + + for temperature in temperatures: + temperature_values.append(float(temperature.data)) + + uwind = dataset[UWIND_LABEL].sel( + longitude=point.x, + latitude=point.y, + isobaricInhPa=temperature[ISOBARIC_LABEL].data, + method="nearest", + ) + uwind_values.append(float(uwind.data)) + + vwind = dataset[VWIND_LABEL].sel( + longitude=point.x, + latitude=point.y, + isobaricInhPa=temperature[ISOBARIC_LABEL].data, + method="nearest", + ) + vwind_values.append(float(vwind.data)) + + cov = Coverage( + id="isobaric", + type="Coverage", + domain=covjson_pydantic.domain.Domain( + domainType=covjson_pydantic.domain.DomainType.vertical_profile, + axes=covjson_pydantic.domain.Axes( + x=covjson_pydantic.domain.ValuesAxis[float](values=[point.x]), + y=covjson_pydantic.domain.ValuesAxis[float](values=[point.y]), + z=covjson_pydantic.domain.ValuesAxis[float](values=isobaric_values), + t=covjson_pydantic.domain.ValuesAxis[AwareDatetime]( + values=[get_temporal_extent(dataset)] + ), + ), + referencing=[ + covjson_pydantic.reference_system.ReferenceSystemConnectionObject( + coordinates=["x", "y"], + system=covjson_pydantic.reference_system.ReferenceSystem( + id="http://www.opengis.net/def/crs/OGC/1.3/CRS84", + type="GeographicCRS", + ), + ), + covjson_pydantic.reference_system.ReferenceSystemConnectionObject( + coordinates=["z"], + system=covjson_pydantic.reference_system.ReferenceSystem( + type="VerticalCRS", + cs={ + "csAxes": [ + { + "name": {"en": "Pressure"}, + "direction": "down", + "unit": {"symbol": "Pa"}, + } + ] + }, + ), + ), + covjson_pydantic.reference_system.ReferenceSystemConnectionObject( + coordinates=["t"], + system=covjson_pydantic.reference_system.ReferenceSystem( + type="TemporalRS", calendar="Gregorian" + ), + ), + ], + ), + ranges={ + "temperature": NdArray( + axisNames=["z"], + shape=[len(isobaric_values)], + values=temperature_values, + ), + "uwind": NdArray( + axisNames=["z"], + shape=[len(isobaric_values)], + values=uwind_values, + ), + "vwind": NdArray( + axisNames=["z"], + shape=[len(isobaric_values)], + values=vwind_values, + ), + }, + parameters={ + "temperature": covjson_pydantic.parameter.Parameter( + id="temperature", + label={"en": "Air temperature"}, + observedProperty=covjson_pydantic.observed_property.ObservedProperty( + id="https://codes.wmo.int/common/quantity-kind/_airTemperature", + label={"en": "Air temperature"}, + ), + unit=covjson_pydantic.unit.Unit( + id="https://codes.wmo.int/common/unit/_K", + label={"en": "Kelvin"}, + symbol="K", + ), + ), + "uwind": covjson_pydantic.parameter.Parameter( + id="uwind", + label={"en": "U component of wind"}, + observedProperty=covjson_pydantic.observed_property.ObservedProperty( + id="https://codes.wmo.int/bufr4/b/11/_095", + label={"en": "u-component of wind"}, + ), + unit=covjson_pydantic.unit.Unit( + id="https://codes.wmo.int/common/unit/_m_s-1", + label={"en": "m/s"}, + symbol="m/s", + ), + ), + "vwind": covjson_pydantic.parameter.Parameter( + id="vwind", + label={"en": "V component of wind"}, + observedProperty=covjson_pydantic.observed_property.ObservedProperty( + id="https://codes.wmo.int/bufr4/b/11/_096", + label={"en": "v-component of wind"}, + ), + unit=covjson_pydantic.unit.Unit( + id="https://codes.wmo.int/common/unit/_m_s-1", + label={"en": "m/s"}, + symbol="m/s", + ), + ), + }, + ) + + return cov.model_dump(exclude_none=True) + + +@router.get("/collections/isobaric/position") +async def create_isobaric_page(coords: str, instance: str = "") -> dict: + """Position. + + This is the main function of this API. Needs a string with the coordinates, and will return data for the nearest point. + + """ + return create_point(coords=coords) + + +@router.get("/collections/isobaric/instances/{instance}/position") +async def create_instance_isobaric_page( + request: Request, coords: str = "", instance: str = "" +) -> dict: + """Position. + + This is the main function of this API. Needs a string with the coordinates, and will return data for the nearest point. + + """ + if len(coords) == 0: + return { + "body": f'Error: No coordinates provided. Example: {str(request.base_url)[0:-1]}{request.scope["path"]}?coords=POINT(11 59)' + } + + return create_point(coords=coords, instance=instance) diff --git a/app/routes/routes.py b/app/routes/routes.py index 897eb0b..b4bf1d3 100644 --- a/app/routes/routes.py +++ b/app/routes/routes.py @@ -10,9 +10,13 @@ landing_page, conformance_page, collections_page, + position_page, + instances_page, ) routes = APIRouter() routes.include_router(landing_page.router) routes.include_router(conformance_page.router) routes.include_router(collections_page.router) +routes.include_router(position_page.router) +routes.include_router(instances_page.router) diff --git a/app/test_app.py b/app/test_app.py index 97ef481..f6041d3 100644 --- a/app/test_app.py +++ b/app/test_app.py @@ -1,4 +1,4 @@ -"""This is mostly to check all URLs reply without crash after code changes.""" +"""Mostly to check all URLs reply without crash after code changes.""" import unittest from fastapi.testclient import TestClient @@ -6,6 +6,7 @@ from app import app client = TestClient(app) +sample_coords = "coords=POINT(11.9384 60.1699)" class TestApp(unittest.TestCase): @@ -31,13 +32,43 @@ def test_collections(self): ) def test_point(self): - response = client.get("/collections/position?coords=POINT(60.1699 11.9384)") + response = client.get(f"/collections/position?{sample_coords}") self.assertEqual(response.status_code, 200) self.assertIn( '"vertical":{"interval":[["850.0"],["100.0"]],"values":["850.0","750.0","700.0","600.0","500.0","450.0","400.0","350.0","300.0","275.0","250.0","225.0","200.0","150.0","100.0"],"vrs":"Vertical Reference System: PressureLevel"}},', response.text, ) + def test_instances(self): + response = client.get("/collections/isobaric/instances") + self.assertEqual(response.status_code, 200) + + json_response = response.json() + instance_id = json_response["instances"][0]["id"] + + self.assertIn( + f"http://localhost:5000/collections/isobaric/instances/{instance_id}/position", + response.text, + ) + + # Test instance + response = client.get("/collections/isobaric/instances/{instance_id}") + self.assertEqual(response.status_code, 200) + self.assertIn( + '"id":"isobaric","title":"', + response.text, + ) + + # Test a point in instance + response = client.get( + f"/collections/isobaric/instances/{instance_id}/position?{sample_coords}" + ) + self.assertEqual(response.status_code, 200) + self.assertIn( + '"id":"isobaric","type":"Coverage","domain":{"type":"Domain","domainType":"VerticalProfile","axes":{"x":{"values', + response.text, + ) + if __name__ == "__main__": unittest.main()