diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 79cd1b5c..9172e493 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,26 +1,6 @@ { - "name": "Ubuntu", - "build": { - "dockerfile": "../docker/Dockerfile", - }, - - // Set *default* container specific settings.json values on container create. - "settings": { - "terminal.integrated.shell.linux": "/bin/bash" - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [], - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - "forwardPorts": [5432], - - // Use 'postCreateCommand' to run commands after the container is created. - //"postCreateCommand": "/docker-entrypoint.sh postgres", - "overrideCommand": false, - - "containerEnv": {"POSTGRES_HOST_AUTH_METHOD": "trust","POSTGRES_USER":"postgres"}, - - // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "root" -} \ No newline at end of file + "name": "PGStac", + "dockerComposeFile": "../docker-compose.yml", + "service": "pgstac", + "workspaceFolder": "/opt/src" +} diff --git a/.dockerignore b/.dockerignore index fdad21c0..b27c26a5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,14 @@ *.eggs venv/* */.direnv/* +*/.ruff_cache/* +*/.vscode/* +*/.mypy_cache/* +*/.pgadmin/* +*/.ipynb_checkpoints/* +*/.git/* +*/.github/* +*/env/* +Dockerfile +docker-compose.yml +*/.devcontainer/* diff --git a/.flake8 b/.flake8 deleted file mode 100644 index cbc2b651..00000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 88 -extend-ignore = E203, W503, E731, E722 -per-file-ignores = __init__.py:F401 diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 30f98373..4fb12504 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -6,11 +6,27 @@ on: - main pull_request: +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + DOCKER_BUILDKIT: 1 + jobs: test: name: test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Execute linters and test suites - run: ./scripts/cibuild + - uses: actions/checkout@v3 + - uses: docker/setup-buildx-action@v1 + - name: builder + id: builder + uses: docker/build-push-action@v2 + with: + context: . + load: true + push: false + cache-from: type=gha + cache-to: type=gha, mode=max + + - name: Run tests + run: docker run --rm ${{ steps.builder.outputs.imageid }} test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 588d3c43..bfaf581b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,14 +24,23 @@ jobs: - name: Install release dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine + pip install setuptools wheel twine build - - name: Build and publish package + - name: Build pypgstac release + run: | + pushd src/pypgstac + rm -rf dist + python -m build --sdist --wheel + popd + + - name: Publish pypgstac release env: TWINE_USERNAME: ${{ secrets.PYPI_STACUTILS_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_STACUTILS_PASSWORD }} run: | - scripts/cipublish + pushd src/pypgstac + twine upload dist/* + popd - name: Tag Release uses: "marvinpinto/action-automatic-releases@v1.2.1" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..ee6f46dd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,61 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: check-yaml + - id: check-added-large-files + - id: check-toml + - id: detect-aws-credentials + args: [--allow-missing-credential] + - id: detect-private-key + - id: check-json + - id: mixed-line-ending + - id: check-merge-conflict + +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: 'v0.0.231' + hooks: + - id: ruff + files: pypgstac\/.*\.py$ + +- repo: local + hooks: + - id: sql + name: sql + entry: scripts/test + args: [--basicsql, --pgtap] + language: script + pass_filenames: false + verbose: true + fail_fast: true + files: sql\/.*\.sql$ + - id: formatting + name: formatting + entry: scripts/test + args: [--formatting] + language: script + pass_filenames: false + verbose: true + fail_fast: true + always_run: true + - id: pypgstac + name: pypgstac + entry: scripts/test + args: [--pypgstac] + language: script + pass_filenames: false + verbose: true + fail_fast: true + files: pypgstac\/.*\.py$ + - id: migrations + name: migrations + entry: scripts/test + args: [--migrations] + language: script + pass_filenames: false + verbose: true + fail_fast: true + files: migrations\/.*\.sql$ diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fa77a18..a2ca5385 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [v0.7.0] + +### Added +- Reorganize code base to create clearer separation between pgstac sql code and pypgstac. +- Move Python tooling to use hatch with all python project configuration in pyproject.toml +- Rework testing framework to not rely on pypgstac or migrations. This allows to run tests on any code updates without creating a version first. If a new version has been staged, the tests will still run through all incremental migrations to make sure they pass as well. +- Add pre-commit to run formatting as well as the tests appropriate for which files have changed. +- Add a query queue to allow for deferred processing of steps that do not change the ability to get results, but enhance performance. The query queue allows to use pg_cron or similar to run tasks that are placed in the queue. +- Modify triggers to allow the use of the query queue for building indexes, adding constraints that are used solely for constraint exclusion, and updating partition and collection spatial and temporal extents. The use of the queue is controlled by the new configuration parameter "use_queue" which can be set as the pgstac.use_queue GUC or by setting in the pgstac_settings table. +- Reorganize how partitions are created and updated to maintain more metadata about partition extents and better tie the constraints to the actual temporal extent of a partition. +- Add "partitions" view that shows stats about number of records, the partition range, constraint ranges, actual date range and spatial extent of each partition. +- Add ability to automatically update the extent object on a collection using the partition metadata via triggers. This is controlled by the new configuration parameter "update_collection_extent" which can be set as the pgstac.update_collection_extent GUC or by setting in the pgstac_settings table. This can be combined with "use_queue" to defer the processing. +- Add many new tests. +- Migrations now make sure that all objects in the pgstac schema are owned by the pgstac_admin role. Functions marked as "SECURITY DEFINER" have been moved to the lower level functions responsible for creating/altering partitions and adding records to the search/search_wheres tables. This should open the door for approaches to using Row Level Security. +- Allow pypgstac loader to load data on pgstac databases that have the same major version even if minor version differs. [162] (https://github.com/stac-utils/pgstac/issues/162) Cherry picked from https://github.com/stac-utils/pgstac/pull/164. + +### Fixed +- Allow empty strings in datetime intervals +- Set search_path and application_name upon connection rather than as kwargs for compatibility with RDS [156] (https://github.com/stac-utils/pgstac/issues/156) + + ## [v0.6.13] ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 256a5420..82fbfd74 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,24 +55,42 @@ scripts/stageversion 0.2.8 This will create a base migration for the new version and will create incremental migrations between any existing base migrations. The incremental migrations that are automatically generated by this script will have the extension ".staged" on the file. You must manually review (and make any modifications necessary) this file and remove the ".staged" extension to enable the migration. ### Making Changes to SQL -All changes to SQL should only be made in the `/sql` directory. SQL Files will be run in alphabetical order. +All changes to SQL should only be made in the `/src/pgstac/sql` directory. SQL Files will be run in alphabetical order. ### Adding Tests -PGStac uses PGTap to test SQL. Tests can be found in tests/pgtap.sql and are run using `scripts/test` +PGStac tests can be written using PGTap or basic SQL output comparisons. Additional testing is available using PyTest in the PyPgSTAC module. Tests can be run using the `scripts/test` command. + +PGTap tests can be written using [PGTap](https://pgtap.org/) syntax. Tests should be added to the `/src/pgstac/tests/pgtap` directory. Any new sql files added to this directory must be added to `/src/pgstac/tests/pgtap.sql`. + +The Basic SQL tests will run any file ending in '.sql' in the `/src/pgstac/tests/basic` directory and will compare the exact results to the corresponding '.sql.out' file. + +PyPgSTAC tests are located in `/src/pypgstac/tests`. + +All tests can be found in tests/pgtap.sql and are run using `scripts/test` + +Individual tests can be run with any combination of the following flags "--formatting --basicsql --pgtap --migrations --pypgstac". If pre-commit is installed, tests will be run on commit based on which files have changed. + + +### To make a PR +1) Make any changes. +2) Make sure there are tests if appropriate. +3) Update Changelog using "### Unreleased" as the version. +4) Make any changes necessary to the docs. +5) Ensure all tests pass (pre-commit will take care of this if installed and the tests will also run on CI) +6) Create PR against the "main" branch. + ### Release Process -1) Make sure all your code is added and committed -2) Create a PR against the main branch -3) Once the PR has been merged, start the release process. -4) Upate the version in `pypgstac/pypgstac/version.py` -5) Use `scripts/stageversion VERSION` as documented in migrations section above making sure to rename any files ending in ".staged" in the migrations section -6) Add details for release to the CHANGELOG -7) Add/Commit any changes -8) Run tests `scripts/test` -9) Create a git tag `git tag v0.2.8` using new version number -10) Push the git tag `git push origin v0.2.8` -11) The CI process will push pypgstac to PyPi, create a docker image on ghcr.io, and create a release on github. +1) Run "scripts/stageversion VERSION" (where version is the next version using semantic versioning ie 0.7.0 +2) Check the incremental migration created in the /src/pgstac/migrations file with the .staged extension to make sure that the generated SQL looks appropriate. +3) Run the tests against the incremental migrations "scripts/test --migrations" +4) Move any "Unreleased" changes in the CHANGELOG.md to the new version. +5) Open a PR for the version change. +6) Once the PR has been merged, start the release process. +7) Create a git tag `git tag v0.2.8` using new version number +8) Push the git tag `git push origin v0.2.8` +9) The CI process will push pypgstac to PyPi, create a docker image on ghcr.io, and create a release on github. ### Get Involved diff --git a/Dockerfile b/Dockerfile index 2ed30185..dcda7544 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,55 +1,41 @@ -FROM postgres:13 as pg - -LABEL maintainer="David Bitner" - +FROM postgres:15-bullseye as pg +ENV PGSTACDOCKER=1 ENV POSTGIS_MAJOR 3 -ENV PGUSER postgres -ENV PGDATABASE postgres -ENV PGHOST localhost -ENV \ - PYTHONUNBUFFERED=1 \ - PYTHONFAULTHANDLER=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 - -RUN \ - apt-get update \ +ENV POSTGIS_VERSION 3.3.2+dfsg-1.pgdg110+1 +ENV PYTHONPATH=/opt/src/pypgstac:/opt/python:${PYTHONPATH} +ENV PATH=/opt/bin:${PATH} +ENV PYTHONWRITEBYTECODE=1 +ENV PYTHONBUFFERED=1 + +RUN set -ex \ + && apt-get update \ && apt-get install -y --no-install-recommends \ - gnupg \ - apt-transport-https \ - debian-archive-keyring \ - software-properties-common \ + ca-certificates \ + python3 python-is-python3 python3-pip \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ postgresql-$PG_MAJOR-pgtap \ postgresql-$PG_MAJOR-partman \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ - build-essential \ - python3 \ - python3-pip \ - python3-setuptools \ - && pip3 install -U pip setuptools packaging \ - && pip3 install -U psycopg2-binary \ - && pip3 install -U psycopg[binary] \ - && pip3 install -U migra[pg] \ && apt-get remove -y apt-transport-https \ - && apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* + && apt-get clean && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /opt/src/pypgstac/pypgstac \ + && touch /opt/src/pypgstac/pypgstac/__init__.py \ + && touch /opt/src/pypgstac/README.md \ + && echo '__version__ = "0.0.0"' > /opt/src/pypgstac/pypgstac/version.py -EXPOSE 5432 +COPY ./src/pypgstac/pyproject.toml /opt/src/pypgstac/pyproject.toml -RUN mkdir -p /docker-entrypoint-initdb.d -RUN echo "#!/bin/bash \n unset PGHOST \n pypgstac migrate" >/docker-entrypoint-initdb.d/initpgstac.sh && chmod +x /docker-entrypoint-initdb.d/initpgstac.sh - -RUN mkdir -p /opt/src/pypgstac - -WORKDIR /opt/src/pypgstac - -COPY pypgstac /opt/src/pypgstac +RUN \ + pip3 install --upgrade pip \ + && pip3 install /opt/src/pypgstac[dev,test,psycopg] -RUN pip3 install -e /opt/src/pypgstac[psycopg] +COPY ./src /opt/src +COPY ./scripts/bin /opt/bin -ENV PYTHONPATH=/opt/src/pypgstac:${PYTHONPATH} +RUN \ + echo "initpgstac" > /docker-entrypoint-initdb.d/999_initpgstac.sh \ + && chmod +x /docker-entrypoint-initdb.d/999_initpgstac.sh \ + && chmod +x /opt/bin/* WORKDIR /opt/src diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index fd3758a1..00000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,25 +0,0 @@ -FROM python:3.8-slim as python-base - -ENV \ - PYTHONUNBUFFERED=1 \ - PYTHONFAULTHANDLER=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PIP_NO_CACHE_DIR=off \ - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 - -RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* - -RUN pip install --upgrade pip && \ - pip install --upgrade psycopg[binary] - -RUN mkdir -p /opt/src/pypgstac - -WORKDIR /opt/src/pypgstac - -COPY pypgstac /opt/src/pypgstac -RUN pip3 install -e /opt/src/pypgstac[dev,psycopg] - -ENV PYTHONPATH=/opt/src/pypgstac:${PYTHONPATH} - -WORKDIR /opt/src diff --git a/README.md b/README.md index eea7e1c0..cf1edccc 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,11 @@ --- -**PgSTAC** is a set of SQL function and schema to build highly performant database for Spatio-Temporal Asset Catalog (STAC). The project also provide **pypgstac** python module to help with the database migration and documents ingestion (collections and items). +**PgSTAC** is a set of SQL function and schema to build highly performant database for Spatio-Temporal Asset Catalog ([STAC](https://stacspec.org)). The project also provide **pypgstac** python module to help with the database migration and documents ingestion (collections and items). + +PgSTAC provides functionality for STAC Filters and CQL2 search along with utilities to help manage indexing and partitioning of STAC Collections and Items. + +PgSTAC is used in production to scale to hundreds of millions of STAC items. PgSTAC implements core data models and functions to provide a STAC API from a PostgreSQL database. As PgSTAC is fully within the database, it does not provide an HTTP facing API. The (Stac FastAPI)[https://github.com/stac-utils/stac-fastapi] PgSTAC backend and (Franklin)[https://github.com/azavea/franklin] can be used to expose a PgSTAC catalog. It is also possible to integrate PgSTAC with any other language that has PostgreSQL drivers. PgSTAC Documentation: https://stac-utils.github.io/pgstac/pgstac @@ -36,10 +40,12 @@ pyPgSTAC Documentation: https://stac-utils.github.io/pgstac/pypgstac ``` / - ├── pypgstac/ - pyPgSTAC python module - ├── scripts/ - scripts to set up the environment - ├── sql/ - PgSTAC SQL code - └── test/ - test suite + ├── src/pypgstac - pyPgSTAC python module + ├── src/pypgstac/tests/ - pyPgSTAC tests + ├── scripts/ - scripts to set up the environment, create migrations, and run tests + ├── src/pgstac/sql/ - PgSTAC SQL code + ├── src/pgstac/migrations/ - Migrations for incremental upgrades + └── src/pgstac/tests/ - test suite ``` ## Contribution & Development diff --git a/docker-compose.yml b/docker-compose.yml index b50b5f2e..8e27e0fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,24 +1,8 @@ version: "3" services: - dev: - container_name: pgstac-dev - image: pgstac-dev - build: - context: . - dockerfile: Dockerfile.dev - platform: linux/amd64 - depends_on: - - database - volumes: - - ./:/opt/src - environment: - - PGUSER=username - - PGPASSWORD=password - - PGHOST=database - - PGDATABASE=postgis - database: - container_name: pgstac-db - image: pgstac-db + pgstac: + container_name: pgstac + image: pgstac build: context: . dockerfile: Dockerfile @@ -29,12 +13,12 @@ services: - POSTGRES_DB=postgis - PGUSER=username - PGPASSWORD=password - - PGHOST=localhost - PGDATABASE=postgis ports: - "5439:5432" volumes: - pgstac-pgdata:/var/lib/postgresql/data - - ./:/opt/src + - ./src:/opt/src + - ./scripts/bin:/opt/bin volumes: pgstac-pgdata: diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 5d9c85c9..00000000 --- a/mypy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[mypy] -ignore_missing_imports = True -disallow_untyped_defs = True -namespace_packages = True diff --git a/pypgstac/poetry.lock b/pypgstac/poetry.lock deleted file mode 100644 index c8a96e13..00000000 --- a/pypgstac/poetry.lock +++ /dev/null @@ -1,759 +0,0 @@ -[[package]] -name = "atomicwrites" -version = "1.4.0" -description = "Atomic file writes." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "attrs" -version = "21.4.0" -description = "Classes Without Boilerplate" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] - -[[package]] -name = "backports.zoneinfo" -version = "0.2.1" -description = "Backport of the standard library zoneinfo module" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -tzdata = ["tzdata"] - -[[package]] -name = "black" -version = "22.3.0" -description = "The uncompromising code formatter." -category = "dev" -optional = false -python-versions = ">=3.6.2" - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -pathspec = ">=0.9.0" -platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - -[[package]] -name = "click" -version = "8.1.2" -description = "Composable command line interface toolkit" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} - -[[package]] -name = "colorama" -version = "0.4.4" -description = "Cross-platform colored terminal text." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "fire" -version = "0.4.0" -description = "A library for automatically generating command line interfaces." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -six = "*" -termcolor = "*" - -[[package]] -name = "flake8" -version = "3.9.2" -description = "the modular source code checker: pep8 pyflakes and co" -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} -mccabe = ">=0.6.0,<0.7.0" -pycodestyle = ">=2.7.0,<2.8.0" -pyflakes = ">=2.3.0,<2.4.0" - -[[package]] -name = "importlib-metadata" -version = "4.11.3" -description = "Read metadata from Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} -zipp = ">=0.5" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] - -[[package]] -name = "mccabe" -version = "0.6.1" -description = "McCabe checker, plugin for flake8" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "more-itertools" -version = "8.12.0" -description = "More routines for operating on iterables, beyond itertools" -category = "dev" -optional = false -python-versions = ">=3.5" - -[[package]] -name = "mypy" -version = "0.910" -description = "Optional static typing for Python" -category = "dev" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -mypy-extensions = ">=0.4.3,<0.5.0" -toml = "*" -typed-ast = {version = ">=1.4.0,<1.5.0", markers = "python_version < \"3.8\""} -typing-extensions = ">=3.7.4" - -[package.extras] -dmypy = ["psutil (>=4.0)"] -python2 = ["typed-ast (>=1.4.0,<1.5.0)"] - -[[package]] -name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "orjson" -version = "3.6.7" -description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "packaging" -version = "21.3" -description = "Core utilities for Python packages" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" - -[[package]] -name = "pathspec" -version = "0.9.0" -description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[[package]] -name = "platformdirs" -version = "2.5.1" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] -test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] - -[[package]] -name = "plpygis" -version = "0.2.0" -description = "PostGIS Python tools" -category = "main" -optional = false -python-versions = "*" - -[package.extras] -shapely_support = ["Shapely (>=1.5.0)"] - -[[package]] -name = "pluggy" -version = "0.13.1" -description = "plugin and hook calling mechanisms for python" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} - -[package.extras] -dev = ["pre-commit", "tox"] - -[[package]] -name = "psycopg" -version = "3.0.11" -description = "PostgreSQL database adapter for Python" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -"backports.zoneinfo" = {version = ">=0.2.0", markers = "python_version < \"3.9\""} -typing-extensions = {version = ">=3.10", markers = "python_version < \"3.8\""} -tzdata = {version = "*", markers = "sys_platform == \"win32\""} - -[package.extras] -binary = ["psycopg-binary (==3.0.11)"] -c = ["psycopg-c (==3.0.11)"] -dev = ["black (>=22.3.0)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=0.920,!=0.930,!=0.931)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] -docs = ["Sphinx (>=4.2)", "furo (==2021.11.23)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)", "dnspython (>=2.1)", "shapely (>=1.7)"] -pool = ["psycopg-pool"] -test = ["mypy (>=0.920,!=0.930,!=0.931)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-asyncio (>=0.16,<0.17)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.10)"] - -[[package]] -name = "psycopg-pool" -version = "3.1.1" -description = "Connection Pool for Psycopg" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "pycodestyle" -version = "2.7.0" -description = "Python style guide checker" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pydantic" -version = "1.9.0" -description = "Data validation and settings management using python 3.6 type hinting" -category = "main" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -typing-extensions = ">=3.7.4.3" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - -[[package]] -name = "pyflakes" -version = "2.3.1" -description = "passive checker of Python programs" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pyparsing" -version = "3.0.7" -description = "Python parsing module" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - -[[package]] -name = "pytest" -version = "5.4.3" -description = "pytest: simple powerful testing with Python" -category = "dev" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} -more-itertools = ">=4.0.0" -packaging = "*" -pluggy = ">=0.12,<1.0" -py = ">=1.5.0" -wcwidth = "*" - -[package.extras] -checkqa-mypy = ["mypy (==v0.761)"] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "smart-open" -version = "4.2.0" -description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" -category = "main" -optional = false -python-versions = ">=3.6.*" - -[package.extras] -all = ["boto3", "google-cloud-storage", "azure-storage-blob", "azure-common", "azure-core", "requests"] -azure = ["azure-storage-blob", "azure-common", "azure-core"] -gcp = ["google-cloud-storage"] -http = ["requests"] -s3 = ["boto3"] -test = ["boto3", "google-cloud-storage", "azure-storage-blob", "azure-common", "azure-core", "requests", "mock", "moto[server] (==1.3.14)", "pathlib2", "responses", "boto3", "paramiko", "parameterizedtestcase", "pytest", "pytest-rerunfailures"] -webhdfs = ["requests"] - -[[package]] -name = "tenacity" -version = "8.0.1" -description = "Retry code until it succeeds" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -doc = ["reno", "sphinx", "tornado (>=4.5)"] - -[[package]] -name = "termcolor" -version = "1.1.0" -description = "ANSII Color formatting for output in terminal." -category = "main" -optional = false -python-versions = "*" - -[[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "dev" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "typed-ast" -version = "1.4.3" -description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "types-orjson" -version = "0.1.1" -description = "Typing stubs for orjson" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "typing-extensions" -version = "4.1.1" -description = "Backported and Experimental Type Hints for Python 3.6+" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "tzdata" -version = "2022.1" -description = "Provider of IANA time zone data" -category = "main" -optional = false -python-versions = ">=2" - -[[package]] -name = "wcwidth" -version = "0.2.5" -description = "Measures the displayed width of unicode strings in a terminal" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "zipp" -version = "3.8.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] - -[metadata] -lock-version = "1.1" -python-versions = ">=3.7" -content-hash = "b8f8aadcfbac840f6ade94cfb16bf02e7a36bb5e54c4471a77e918e24013c6f9" - -[metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, -] -attrs = [ - {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, - {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, -] -"backports.zoneinfo" = [ - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc"}, - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722"}, - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546"}, - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win32.whl", hash = "sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08"}, - {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win32.whl", hash = "sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b"}, - {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-win32.whl", hash = "sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328"}, - {file = "backports.zoneinfo-0.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6"}, - {file = "backports.zoneinfo-0.2.1.tar.gz", hash = "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2"}, -] -black = [ - {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, - {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, - {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, - {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, - {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, - {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, - {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, - {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, - {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, - {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, - {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, - {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, - {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, - {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, - {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, - {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, - {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, - {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, - {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, - {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, - {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, - {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, - {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, -] -click = [ - {file = "click-8.1.2-py3-none-any.whl", hash = "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e"}, - {file = "click-8.1.2.tar.gz", hash = "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72"}, -] -colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, -] -fire = [ - {file = "fire-0.4.0.tar.gz", hash = "sha256:c5e2b8763699d1142393a46d0e3e790c5eb2f0706082df8f647878842c216a62"}, -] -flake8 = [ - {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"}, - {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"}, -] -importlib-metadata = [ - {file = "importlib_metadata-4.11.3-py3-none-any.whl", hash = "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6"}, - {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, -] -mccabe = [ - {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, - {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, -] -more-itertools = [ - {file = "more-itertools-8.12.0.tar.gz", hash = "sha256:7dc6ad46f05f545f900dd59e8dfb4e84a4827b97b3cfecb175ea0c7d247f6064"}, - {file = "more_itertools-8.12.0-py3-none-any.whl", hash = "sha256:43e6dd9942dffd72661a2c4ef383ad7da1e6a3e968a927ad7a6083ab410a688b"}, -] -mypy = [ - {file = "mypy-0.910-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457"}, - {file = "mypy-0.910-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb"}, - {file = "mypy-0.910-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9"}, - {file = "mypy-0.910-cp35-cp35m-win_amd64.whl", hash = "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e"}, - {file = "mypy-0.910-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921"}, - {file = "mypy-0.910-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6"}, - {file = "mypy-0.910-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212"}, - {file = "mypy-0.910-cp36-cp36m-win_amd64.whl", hash = "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885"}, - {file = "mypy-0.910-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0"}, - {file = "mypy-0.910-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de"}, - {file = "mypy-0.910-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703"}, - {file = "mypy-0.910-cp37-cp37m-win_amd64.whl", hash = "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a"}, - {file = "mypy-0.910-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504"}, - {file = "mypy-0.910-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9"}, - {file = "mypy-0.910-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072"}, - {file = "mypy-0.910-cp38-cp38-win_amd64.whl", hash = "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811"}, - {file = "mypy-0.910-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e"}, - {file = "mypy-0.910-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b"}, - {file = "mypy-0.910-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2"}, - {file = "mypy-0.910-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97"}, - {file = "mypy-0.910-cp39-cp39-win_amd64.whl", hash = "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8"}, - {file = "mypy-0.910-py3-none-any.whl", hash = "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"}, - {file = "mypy-0.910.tar.gz", hash = "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150"}, -] -mypy-extensions = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, -] -orjson = [ - {file = "orjson-3.6.7-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:93188a9d6eb566419ad48befa202dfe7cd7a161756444b99c4ec77faea9352a4"}, - {file = "orjson-3.6.7-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:82515226ecb77689a029061552b5df1802b75d861780c401e96ca6bc8495f775"}, - {file = "orjson-3.6.7-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3af57ffab7848aaec6ba6b9e9b41331250b57bf696f9d502bacdc71a0ebab0ba"}, - {file = "orjson-3.6.7-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:a7297504d1142e7efa236ffc53f056d73934a993a08646dbcee89fc4308a8fcf"}, - {file = "orjson-3.6.7-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:5a50cde0dbbde255ce751fd1bca39d00ecd878ba0903c0480961b31984f2fab7"}, - {file = "orjson-3.6.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d21f9a2d1c30e58070f93988db4cad154b9009fafbde238b52c1c760e3607fbe"}, - {file = "orjson-3.6.7-cp310-none-win_amd64.whl", hash = "sha256:e152464c4606b49398afd911777decebcf9749cc8810c5b4199039e1afb0991e"}, - {file = "orjson-3.6.7-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:0a65f3c403f38b0117c6dd8e76e85a7bd51fcd92f06c5598dfeddbc44697d3e5"}, - {file = "orjson-3.6.7-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:6c47cfca18e41f7f37b08ff3e7abf5ada2d0f27b5ade934f05be5fc5bb956e9d"}, - {file = "orjson-3.6.7-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:63185af814c243fad7a72441e5f98120c9ecddf2675befa486d669fb65539e9b"}, - {file = "orjson-3.6.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2da6fde42182b80b40df2e6ab855c55090ebfa3fcc21c182b7ad1762b61d55c"}, - {file = "orjson-3.6.7-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:48c5831ec388b4e2682d4ff56d6bfa4a2ef76c963f5e75f4ff4785f9cf338a80"}, - {file = "orjson-3.6.7-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:913fac5d594ccabf5e8fbac15b9b3bb9c576d537d49eeec9f664e7a64dde4c4b"}, - {file = "orjson-3.6.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:58f244775f20476e5851e7546df109f75160a5178d44257d437ba6d7e562bfe8"}, - {file = "orjson-3.6.7-cp37-none-win_amd64.whl", hash = "sha256:2d5f45c6b85e5f14646df2d32ecd7ff20fcccc71c0ea1155f4d3df8c5299bbb7"}, - {file = "orjson-3.6.7-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:612d242493afeeb2068bc72ff2544aa3b1e627578fcf92edee9daebb5893ffea"}, - {file = "orjson-3.6.7-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:539cdc5067db38db27985e257772d073cd2eb9462d0a41bde96da4e4e60bd99b"}, - {file = "orjson-3.6.7-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6d103b721bbc4f5703f62b3882e638c0b65fcdd48622531c7ffd45047ef8e87c"}, - {file = "orjson-3.6.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb10a20f80e95102dd35dfbc3a22531661b44a09b55236b012a446955846b023"}, - {file = "orjson-3.6.7-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:bb68d0da349cf8a68971a48ad179434f75256159fe8b0715275d9b49fa23b7a3"}, - {file = "orjson-3.6.7-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:4a2c7d0a236aaeab7f69c17b7ab4c078874e817da1bfbb9827cb8c73058b3050"}, - {file = "orjson-3.6.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3be045ca3b96119f592904cf34b962969ce97bd7843cbfca084009f6c8d2f268"}, - {file = "orjson-3.6.7-cp38-none-win_amd64.whl", hash = "sha256:bd765c06c359d8a814b90f948538f957fa8a1f55ad1aaffcdc5771996aaea061"}, - {file = "orjson-3.6.7-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:7dd9e1e46c0776eee9e0649e3ae9584ea368d96851bcaeba18e217fa5d755283"}, - {file = "orjson-3.6.7-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:c4b4f20a1e3df7e7c83717aff0ef4ab69e42ce2fb1f5234682f618153c458406"}, - {file = "orjson-3.6.7-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7107a5673fd0b05adbb58bf71c1578fc84d662d29c096eb6d998982c8635c221"}, - {file = "orjson-3.6.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a08b6940dd9a98ccf09785890112a0f81eadb4f35b51b9a80736d1725437e22c"}, - {file = "orjson-3.6.7-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:f5d1648e5a9d1070f3628a69a7c6c17634dbb0caf22f2085eca6910f7427bf1f"}, - {file = "orjson-3.6.7-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:e6201494e8dff2ce7fd21da4e3f6dfca1a3fed38f9dcefc972f552f6596a7621"}, - {file = "orjson-3.6.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:70d0386abe02879ebaead2f9632dd2acb71000b4721fd8c1a2fb8c031a38d4d5"}, - {file = "orjson-3.6.7-cp39-none-win_amd64.whl", hash = "sha256:d9a3288861bfd26f3511fb4081561ca768674612bac59513cb9081bb61fcc87f"}, - {file = "orjson-3.6.7.tar.gz", hash = "sha256:a4bb62b11289b7620eead2f25695212e9ac77fcfba76f050fa8a540fb5c32401"}, -] -packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, -] -pathspec = [ - {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, - {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, -] -platformdirs = [ - {file = "platformdirs-2.5.1-py3-none-any.whl", hash = "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227"}, - {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"}, -] -plpygis = [ - {file = "plpygis-0.2.0.tar.gz", hash = "sha256:f9d1bb3913970b6c40c67188be3716f9fa490c1441e6c0d915221c8291826079"}, -] -pluggy = [ - {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, - {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, -] -psycopg = [ - {file = "psycopg-3.0.11-py3-none-any.whl", hash = "sha256:8fcbac023ef84922e107645fa8e21b7dc3b0fec0ca49c4270b53d7e64fb08e5f"}, - {file = "psycopg-3.0.11.tar.gz", hash = "sha256:5dfe409eefc91890602dff18ee17b3815e803d48f87de48f3fa8587653a5a2fb"}, -] -psycopg-pool = [ - {file = "psycopg-pool-3.1.1.tar.gz", hash = "sha256:bc579078dc8209f1ce280228460f96770756f24babb5d8ab2418800e9082a973"}, - {file = "psycopg_pool-3.1.1-py3-none-any.whl", hash = "sha256:397beaa082f17255e6267850a00700aec4427fa214b4c55d2d49c7c154508ed5"}, -] -py = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, -] -pycodestyle = [ - {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"}, - {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"}, -] -pydantic = [ - {file = "pydantic-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cb23bcc093697cdea2708baae4f9ba0e972960a835af22560f6ae4e7e47d33f5"}, - {file = "pydantic-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1d5278bd9f0eee04a44c712982343103bba63507480bfd2fc2790fa70cd64cf4"}, - {file = "pydantic-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab624700dc145aa809e6f3ec93fb8e7d0f99d9023b713f6a953637429b437d37"}, - {file = "pydantic-1.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8d7da6f1c1049eefb718d43d99ad73100c958a5367d30b9321b092771e96c25"}, - {file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3c3b035103bd4e2e4a28da9da7ef2fa47b00ee4a9cf4f1a735214c1bcd05e0f6"}, - {file = "pydantic-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3011b975c973819883842c5ab925a4e4298dffccf7782c55ec3580ed17dc464c"}, - {file = "pydantic-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:086254884d10d3ba16da0588604ffdc5aab3f7f09557b998373e885c690dd398"}, - {file = "pydantic-1.9.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0fe476769acaa7fcddd17cadd172b156b53546ec3614a4d880e5d29ea5fbce65"}, - {file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8e9dcf1ac499679aceedac7e7ca6d8641f0193c591a2d090282aaf8e9445a46"}, - {file = "pydantic-1.9.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1e4c28f30e767fd07f2ddc6f74f41f034d1dd6bc526cd59e63a82fe8bb9ef4c"}, - {file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c86229333cabaaa8c51cf971496f10318c4734cf7b641f08af0a6fbf17ca3054"}, - {file = "pydantic-1.9.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:c0727bda6e38144d464daec31dff936a82917f431d9c39c39c60a26567eae3ed"}, - {file = "pydantic-1.9.0-cp36-cp36m-win_amd64.whl", hash = "sha256:dee5ef83a76ac31ab0c78c10bd7d5437bfdb6358c95b91f1ba7ff7b76f9996a1"}, - {file = "pydantic-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9c9bdb3af48e242838f9f6e6127de9be7063aad17b32215ccc36a09c5cf1070"}, - {file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ee7e3209db1e468341ef41fe263eb655f67f5c5a76c924044314e139a1103a2"}, - {file = "pydantic-1.9.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b6037175234850ffd094ca77bf60fb54b08b5b22bc85865331dd3bda7a02fa1"}, - {file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b2571db88c636d862b35090ccf92bf24004393f85c8870a37f42d9f23d13e032"}, - {file = "pydantic-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8b5ac0f1c83d31b324e57a273da59197c83d1bb18171e512908fe5dc7278a1d6"}, - {file = "pydantic-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:bbbc94d0c94dd80b3340fc4f04fd4d701f4b038ebad72c39693c794fd3bc2d9d"}, - {file = "pydantic-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e0896200b6a40197405af18828da49f067c2fa1f821491bc8f5bde241ef3f7d7"}, - {file = "pydantic-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bdfdadb5994b44bd5579cfa7c9b0e1b0e540c952d56f627eb227851cda9db77"}, - {file = "pydantic-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:574936363cd4b9eed8acdd6b80d0143162f2eb654d96cb3a8ee91d3e64bf4cf9"}, - {file = "pydantic-1.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c556695b699f648c58373b542534308922c46a1cda06ea47bc9ca45ef5b39ae6"}, - {file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f947352c3434e8b937e3aa8f96f47bdfe6d92779e44bb3f41e4c213ba6a32145"}, - {file = "pydantic-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5e48ef4a8b8c066c4a31409d91d7ca372a774d0212da2787c0d32f8045b1e034"}, - {file = "pydantic-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:96f240bce182ca7fe045c76bcebfa0b0534a1bf402ed05914a6f1dadff91877f"}, - {file = "pydantic-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:815ddebb2792efd4bba5488bc8fde09c29e8ca3227d27cf1c6990fc830fd292b"}, - {file = "pydantic-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c5b77947b9e85a54848343928b597b4f74fc364b70926b3c4441ff52620640c"}, - {file = "pydantic-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c68c3bc88dbda2a6805e9a142ce84782d3930f8fdd9655430d8576315ad97ce"}, - {file = "pydantic-1.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a79330f8571faf71bf93667d3ee054609816f10a259a109a0738dac983b23c3"}, - {file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f5a64b64ddf4c99fe201ac2724daada8595ada0d102ab96d019c1555c2d6441d"}, - {file = "pydantic-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a733965f1a2b4090a5238d40d983dcd78f3ecea221c7af1497b845a9709c1721"}, - {file = "pydantic-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cc6a4cb8a118ffec2ca5fcb47afbacb4f16d0ab8b7350ddea5e8ef7bcc53a16"}, - {file = "pydantic-1.9.0-py3-none-any.whl", hash = "sha256:085ca1de245782e9b46cefcf99deecc67d418737a1fd3f6a4f511344b613a5b3"}, - {file = "pydantic-1.9.0.tar.gz", hash = "sha256:742645059757a56ecd886faf4ed2441b9c0cd406079c2b4bee51bcc3fbcd510a"}, -] -pyflakes = [ - {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"}, - {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"}, -] -pyparsing = [ - {file = "pyparsing-3.0.7-py3-none-any.whl", hash = "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"}, - {file = "pyparsing-3.0.7.tar.gz", hash = "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea"}, -] -pytest = [ - {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"}, - {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -smart-open = [ - {file = "smart_open-4.2.0.tar.gz", hash = "sha256:d9f5a0f173ccb9bbae528db5a3804f57145815774f77ef755b9b0f3b4b2a9dcb"}, -] -tenacity = [ - {file = "tenacity-8.0.1-py3-none-any.whl", hash = "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a"}, - {file = "tenacity-8.0.1.tar.gz", hash = "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f"}, -] -termcolor = [ - {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, -] -toml = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -typed-ast = [ - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, - {file = "typed_ast-1.4.3-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win32.whl", hash = "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428"}, - {file = "typed_ast-1.4.3-cp35-cp35m-win_amd64.whl", hash = "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3"}, - {file = "typed_ast-1.4.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace"}, - {file = "typed_ast-1.4.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win32.whl", hash = "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363"}, - {file = "typed_ast-1.4.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7"}, - {file = "typed_ast-1.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04"}, - {file = "typed_ast-1.4.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win32.whl", hash = "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c"}, - {file = "typed_ast-1.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805"}, - {file = "typed_ast-1.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41"}, - {file = "typed_ast-1.4.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39"}, - {file = "typed_ast-1.4.3-cp38-cp38-win32.whl", hash = "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927"}, - {file = "typed_ast-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40"}, - {file = "typed_ast-1.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0"}, - {file = "typed_ast-1.4.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3"}, - {file = "typed_ast-1.4.3-cp39-cp39-win32.whl", hash = "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808"}, - {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, - {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, -] -types-orjson = [ - {file = "types-orjson-0.1.1.tar.gz", hash = "sha256:7454bfbaed27900a844bb9d8e211b69f1c335f0b9e3541d4950a793db41c104d"}, - {file = "types_orjson-0.1.1-py2.py3-none-any.whl", hash = "sha256:92f85986261ea1a5cb215e4b35e4016631d35163a372f023918750f340ea737f"}, -] -typing-extensions = [ - {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"}, - {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"}, -] -tzdata = [ - {file = "tzdata-2022.1-py2.py3-none-any.whl", hash = "sha256:238e70234214138ed7b4e8a0fab0e5e13872edab3be586ab8198c407620e2ab9"}, - {file = "tzdata-2022.1.tar.gz", hash = "sha256:8b536a8ec63dc0751342b3984193a3118f8fca2afe25752bb9b7fffd398552d3"}, -] -wcwidth = [ - {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, - {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, -] -zipp = [ - {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, - {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, -] diff --git a/pypgstac/pypgstac/version.py b/pypgstac/pypgstac/version.py deleted file mode 100644 index 09ea5e3d..00000000 --- a/pypgstac/pypgstac/version.py +++ /dev/null @@ -1,2 +0,0 @@ -"""Version.""" -__version__ = "0.6.13" diff --git a/pypgstac/setup.cfg b/pypgstac/setup.cfg deleted file mode 100644 index b7314f83..00000000 --- a/pypgstac/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[metadata] -version = attr: pypgstac.version.__version__ - -[options.entry_points] -console_scripts = pypgstac=pypgstac.pypgstac:cli diff --git a/pypgstac/setup.py b/pypgstac/setup.py deleted file mode 100644 index 9bbf663f..00000000 --- a/pypgstac/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -"""pypgstac: python utilities for working with pgstac.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "smart-open[html]>=4.2,<7.0", - "orjson>=3.5.2", - "python-dateutil==2.8.*", - "fire==0.4.*", - "plpygis==0.2.*", - "pydantic[dotenv]==1.10.*", - "tenacity==8.1.*", -] - -extra_reqs = { - "dev": [ - "pytest==5.*", - "flake8==3.9.*", - "black>=21.7b0", - "mypy>=0.910", - "types-orjson==0.1.1", - "pystac[validation]==1.*" - ], - "psycopg": [ - "psycopg[binary]==3.1.*", - "psycopg-pool==3.1.*", - ], -} - - -setup( - name="pypgstac", - description="Schema, functions and a python library for storing and accessing STAC collections and items in PostgreSQL", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.7", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.7", - "License :: OSI Approved :: MIT License", - ], - keywords="stac, postgres", - author="David Bitner", - author_email="bitner@dbspatial.com", - url="https://github.com/stac-utils/pgstac", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["migrations/pgstac*.sql", "py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=[extra_reqs["dev"], extra_reqs["psycopg"]], - extras_require=extra_reqs, -) diff --git a/pypgstac/tests/conftest.py b/pypgstac/tests/conftest.py deleted file mode 100644 index f04455c0..00000000 --- a/pypgstac/tests/conftest.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Fixtures for pypgstac tests.""" -from typing import Generator -import pytest -import os -import psycopg -from pypgstac.db import PgstacDB -from pypgstac.migrate import Migrate -from pypgstac.load import Loader - - -@pytest.fixture(scope="function") -def db() -> Generator: - """Fixture to get a fresh database.""" - origdb: str = os.getenv("PGDATABASE", "") - - with psycopg.connect(autocommit=True) as conn: - try: - conn.execute("CREATE DATABASE pgstactestdb;") - except psycopg.errors.DuplicateDatabase: - try: - conn.execute("DROP DATABASE pgstactestdb WITH (FORCE);") - conn.execute("CREATE DATABASE pgstactestdb;") - except psycopg.errors.InsufficientPrivilege: - try: - conn.execute("DROP DATABASE pgstactestdb;") - conn.execute("CREATE DATABASE pgstactestdb;") - except: - pass - - os.environ["PGDATABASE"] = "pgstactestdb" - - pgdb = PgstacDB() - - yield pgdb - - print("Closing Connection and Dropping DB") - pgdb.close() - os.environ["PGDATABASE"] = origdb - - with psycopg.connect(autocommit=True) as conn: - try: - conn.execute("DROP DATABASE pgstactestdb WITH (FORCE);") - except psycopg.errors.InsufficientPrivilege: - try: - conn.execute("DROP DATABASE pgstactestdb;") - except: - pass - - -@pytest.fixture(scope="function") -def loader(db: PgstacDB) -> Generator: - """Fixture to get a loader and an empty pgstac.""" - db.query("DROP SCHEMA IF EXISTS pgstac CASCADE;") - migrator = Migrate(db) - print(migrator.run_migration()) - ldr = Loader(db) - yield ldr diff --git a/scripts/bin/README b/scripts/bin/README new file mode 100644 index 00000000..5359a42f --- /dev/null +++ b/scripts/bin/README @@ -0,0 +1 @@ +scripts/bin contains scripts that are meant to be run within the docker container diff --git a/scripts/bin/format b/scripts/bin/format index 2aac5936..18bb532d 100755 --- a/scripts/bin/format +++ b/scripts/bin/format @@ -5,6 +5,10 @@ set -e if [[ "${CI}" ]]; then set -x fi +if [ ! $PGSTACDOCKER == 1 ]; then + echo "This script should only be run within pgstac docker"; exit 1; +fi +cd /opt/src/ function usage() { echo -n \ @@ -18,6 +22,6 @@ This scripts is meant to be run inside the dev container. if [ "${BASH_SOURCE[0]}" = "${0}" ]; then echo "Formatting pypgstac..." - black pypgstac/pypgstac - black pypgstac/tests + ruff --fix pypgstac/pypgstac + ruff --fix pypgstac/tests fi diff --git a/scripts/bin/initpgstac b/scripts/bin/initpgstac new file mode 100755 index 00000000..e937eded --- /dev/null +++ b/scripts/bin/initpgstac @@ -0,0 +1,11 @@ +#!/bin/bash +if [ ! $PGSTACDOCKER == 1 ]; then + echo "This script should only be run within pgstac docker"; exit 1; +fi +cd /opt/src/pgstac +psql -X -q -v ON_ERROR_STOP=1 </dev/null 2>&1 <<-'EOSQL' - DROP DATABASE IF EXISTS migra_from; - CREATE DATABASE migra_from; - DROP DATABASE IF EXISTS migra_to; - CREATE DATABASE migra_to; -EOSQL -} -export -f create_migra_dbs - -function drop_migra_dbs(){ -psql -q >/dev/null 2>&1 <<-'EOSQL' - DROP DATABASE IF EXISTS migra_from; - DROP DATABASE IF EXISTS migra_to; -EOSQL -} -export -f drop_migra_dbs - -function pgwait(){ - RETRIES=10 - until pg_isready >/dev/null 2>&1 || [ $RETRIES -eq 0 ]; do - sleep 1 - done -} -export -f pgwait - -function pgtap(){ - TESTOUTPUT=$(psql -X -f $BASEDIR/test/pgtap.sql $1) - echo "Checking if any tests are not ok on db $1" - if [[ $(echo "$TESTOUTPUT" | grep -e '^not') ]]; then - echo "PGTap tests failed." - echo "$TESTOUTPUT" - exit 1 - else - echo "All PGTap Tests Passed!" - fi -} -export -f pgtap - -function calc_migration(){ - cd $MIGRATIONS_DIR - tmpfile=$(mktemp) - trap "rm -f $tmpfile" 0 2 3 15 - MIGRA_FROM_FILE=$1 - MIGRA_TO_FILE=$2 - pgwait - create_migra_dbs - trap drop_migra_dbs 0 2 3 15 - - psql -q -X -1 -v ON_ERROR_STOP=1 -v CLIENT_MIN_MESSAGES=WARNING -f $MIGRA_FROM_FILE $FROMDBURL >/dev/null || exit 1; - psql -q -X -1 -v ON_ERROR_STOP=1 -v CLIENT_MIN_MESSAGES=WARNING -f $MIGRA_TO_FILE $TODBURL >/dev/null || exit 1; - - migra --schema pgstac --unsafe $FROMDBURL $TODBURL || echo "" -} -export -f calc_migration diff --git a/scripts/bin/resetpgstac b/scripts/bin/resetpgstac new file mode 100755 index 00000000..29c01839 --- /dev/null +++ b/scripts/bin/resetpgstac @@ -0,0 +1,14 @@ +#!/bin/bash +if [ ! $PGSTACDOCKER == 1 ]; then + echo "This script should only be run within pgstac docker"; exit 1; +fi +cd /opt/src/pgstac +set -e +psql -f pgstac.sql +psql -v ON_ERROR_STOP=1 <<-EOSQL + DROP SCHEMA IF EXISTS pgstac CASCADE; + \i pgstac.sql + SET SEARCH_PATH TO pgstac, public; + \copy collections (content) FROM 'tests/testdata/collections.ndjson' + \copy items_staging (content) FROM 'tests/testdata/items.ndjson' +EOSQL diff --git a/scripts/bin/stageversion b/scripts/bin/stageversion index b1ec39bf..55ef0bcc 100755 --- a/scripts/bin/stageversion +++ b/scripts/bin/stageversion @@ -1,23 +1,32 @@ #!/bin/bash -source $(dirname $0)/migra_funcs set -e if [[ "${CI}" ]]; then set -x fi +if [ ! $PGSTACDOCKER == 1 ]; then + echo "This script should only be run within pgstac docker"; exit 1; +fi + +source $(dirname $0)/tmpdb +setuptestdb + +BASEDIR=/opt/src +PYPGSTACDIR=$BASEDIR/pypgstac +MIGRATIONSDIR=$BASEDIR/pgstac/migrations + function usage() { echo -n \ - "Usage: $(basename "$0") VERSION -Stage migrations for VERSION + "Usage: $(basename "$0"). VERSION FORCE This scripts is meant to be run inside the dev container. " } - VERSION=$1 - +FORCE=$2 +[ -n "${FORCE}" ] && echo "Rewriting last migration step directly" || echo "Creating staging file for last migration." if [[ -z "${VERSION}" ]]; then echo "ERROR: Must supply a version." usage @@ -25,12 +34,107 @@ if [[ -z "${VERSION}" ]]; then fi -if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - # Modify 999_version.sql so that base migrations have a reference to the version. - echo "SELECT set_version('${VERSION}');" >sql/999_version.sql +TODBURL="postgresql://${PGUSER}:${PGPASSWORD}@${PGHOST:-localhost}:${PGPORT:-5432}/migra_to" +FROMDBURL="postgresql://${PGUSER}:${PGPASSWORD}@${PGHOST:-localhost}:${PGPORT:-5432}/migra_from" - # Assemble a base migration for the version and put it in the migrations directory. - cat sql/*.sql >pypgstac/pypgstac/migrations/pgstac.${VERSION}.sql +echo $TODBURL +export | grep PG - echo $VERSION -fi +function base_migrations(){ + find $MIGRATIONSDIR -regex ".*\/pgstac\.[0-9]+\.[0-9]+\.[0-9]+\.sql" +} + +function base_migration_versions(){ + base_migrations | sed -En 's/.*pgstac\.([0-9]+\.[0-9]+\.[0-9]+)\.sql/\1/p' | sort -V +} + +function create_migra_dbs(){ +psql -q >/dev/null 2>&1 <<-'EOSQL' + DROP DATABASE IF EXISTS migra_from; + CREATE DATABASE migra_from; + DROP DATABASE IF EXISTS migra_to; + CREATE DATABASE migra_to; + DROP DATABASE IF EXISTS base_test; + CREATE DATABASE base_test; +EOSQL +} + +function drop_migra_dbs(){ +psql -q >/dev/null 2>&1 <<-'EOSQL' + DROP DATABASE IF EXISTS migra_from; + DROP DATABASE IF EXISTS migra_to; +EOSQL +} + +function pgwait(){ + RETRIES=10 + until pg_isready >/dev/null 2>&1 || [ $RETRIES -eq 0 ]; do + sleep 1 + done +} + +function calc_migration(){ + cd $MIGRATIONS_DIR + tmpfile=$(mktemp) + trap "rm -f $tmpfile" 0 2 3 15 + MIGRA_FROM_FILE=$1 + MIGRA_TO_FILE=$2 + pgwait + create_migra_dbs + trap drop_migra_dbs 0 2 3 15 + + psql -q -X -1 -v ON_ERROR_STOP=1 -v CLIENT_MIN_MESSAGES=WARNING -f $MIGRA_FROM_FILE $FROMDBURL >/dev/null || exit 1; + psql -q -X -1 -v ON_ERROR_STOP=1 -v CLIENT_MIN_MESSAGES=WARNING -f $MIGRA_TO_FILE $TODBURL >/dev/null || exit 1; + + migra --schema pgstac --unsafe $FROMDBURL $TODBURL || echo "" +} + +cd $BASEDIR/pgstac/sql + +echo "SELECT set_version('${VERSION}');" >999_version.sql + +# Assemble a base migration for the version and put it in the migrations directory. +cat *.sql >$MIGRATIONSDIR/pgstac.${VERSION}.sql + + +cat < $PYPGSTACDIR/pypgstac/version.py +"""Version.""" +__version__ = "${VERSION}" +EOD + + +# Get Array of available base migration files +readarray -t VERSIONS < <(base_migration_versions) + +# Calculate incremental versions sql migrations +cnt=$((${#VERSIONS[@]}-1)) +for (( i=0; i<$cnt; i++ )); do + F=${VERSIONS[$i]} + F_BASE="$MIGRATIONSDIR/pgstac.${F}.sql" + T=${VERSIONS[$i+1]} + T_BASE="$MIGRATIONSDIR/pgstac.${T}.sql" + + FILE="$MIGRATIONSDIR/pgstac.${F}-${T}.sql" + [ -n "${FORCE}" -a -f "$FILE" -a $(( $i + 1 )) -eq $cnt ] && rm $FILE + STAGED="$FILE.staged" + STAGEDINIT="$STAGED.init" + trap "rm $STAGEDINIT" 0 2 3 15 + + if [ -f $FILE ]; then + echo "Migration $FILE already exists." + else + echo "Creating migrations from $F to $T" + echo "SET client_min_messages TO WARNING;" >$STAGEDINIT + echo "SET SEARCH_PATH to pgstac, public;" >>$STAGEDINIT + echo "-- BEGIN migra calculated SQL" >>$STAGEDINIT + calc_migration $F_BASE $T_BASE >>$STAGEDINIT + echo "-- END migra calculated SQL" >>$STAGEDINIT + cd /opt/src/pgstac/sql + cat 000_idempotent_pre.sql $STAGEDINIT 998_idempotent_post.sql 999_version.sql >$STAGED + rm $STAGEDINIT + echo "Created '$STAGED'. You must review and rename to $FILE before" + echo "committing and tagging a release." + fi + [ -n "${FORCE}" -a -f "$STAGED" ] && mv $STAGED $FILE +done +exit 0 diff --git a/scripts/bin/stageversiondb b/scripts/bin/stageversiondb deleted file mode 100755 index 47bb8c5c..00000000 --- a/scripts/bin/stageversiondb +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -source $(dirname $0)/migra_funcs -set -e - -if [[ "${CI}" ]]; then - set -x -fi - -function usage() { - echo -n \ - "Usage: $(basename "$0"). - -This scripts is meant to be run inside the dev container. - -" -} - -FORCE=$1 -[ -n "${FORCE}" ] && echo "Rewriting last migration step directly" || echo "Creating staging file for last migration." - -if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - cd $BASEDIR - # Get Array of available base migration files - readarray -t VERSIONS < <(base_migration_versions) - - # Calculate incremental versions sql migrations - cnt=$((${#VERSIONS[@]}-1)) - for (( i=0; i<$cnt; i++ )); do - F=${VERSIONS[$i]} - F_BASE="$MIGRATIONSDIR/pgstac.${F}.sql" - T=${VERSIONS[$i+1]} - T_BASE="$MIGRATIONSDIR/pgstac.${T}.sql" - - FILE="$MIGRATIONSDIR/pgstac.${F}-${T}.sql" - [ -n "${FORCE}" -a -f "$FILE" -a $(( $i + 1 )) -eq $cnt ] && rm $FILE - STAGED="$FILE.staged" - - if [ -f $FILE ]; then - echo "Migration $FILE already exists." - else - echo "Creating migrations from $F to $T" - echo "SET client_min_messages TO WARNING;" >$STAGED - echo "SET SEARCH_PATH to pgstac, public;" >>$STAGED - echo $? - echo $F_BASE $T_BASE - calc_migration $F_BASE $T_BASE >>$STAGED - echo $? - echo "SELECT set_version('$T');" >>$STAGED - echo $? - echo "Created '$STAGED'. You must review and rename to $FILE before" - echo "committing and tagging a release." - fi - [ -n "${FORCE}" -a -f "$STAGED" ] && mv $STAGED $FILE - done -fi diff --git a/scripts/bin/test b/scripts/bin/test index e241160a..35711836 100755 --- a/scripts/bin/test +++ b/scripts/bin/test @@ -1,44 +1,208 @@ #!/bin/bash - set -e if [[ "${CI}" ]]; then set -x fi +if [ ! $PGSTACDOCKER == 1 ]; then + echo "This script should only be run within pgstac docker"; exit 1; +fi + +source $(dirname $0)/tmpdb function usage() { echo -n \ "Usage: $(basename "$0") -Runs tests for the project. - +Run PgSTAC tests. This scripts is meant to be run inside the dev container. " } -if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - - echo "Running mypy..." - mypy pypgstac/pypgstac pypgstac/tests - echo "Running black..." - black --check pypgstac/pypgstac pypgstac/tests +function test_formatting(){ + cd /opt/src/pypgstac - echo "Running flake8..." - flake8 pypgstac/pypgstac pypgstac/tests + echo "Running ruff" + ruff --cache-dir=/tmp/.ruff pypgstac tests - echo "Running unit tests..." - pytest pypgstac/tests + echo "Running mypy" + mypy --cache-dir=/tmp/.mypy pypgstac echo "Checking if there are any staged migrations." - find /opt/src/pypgstac/pypgstac/migrations | grep 'staged' && { echo "There are staged migrations in pypgstac/pypgstac/migrations. Please check migrations and remove staged suffix."; exit 1; } + find pypgstac/migrations | grep 'staged' && { echo "There are staged migrations in pypgstac/migrations. Please check migrations and remove staged suffix."; exit 1; } + + + VERSION=$(python -c "from pypgstac.version import __version__; print(__version__)") + echo $VERSION + + echo "Checking whether base sql migration exists for pypgstac version." + [ -f pypgstac/migrations/pgstac."${VERSION}".sql ] || { echo "****FAIL No Migration exists pypgstac/migrations/pgstac.${VERSION}.sql"; exit 1; } + + echo "Congratulations! All formatting tests pass." +} +function test_pgtap(){ +cd /opt/src/pgstac +TEMPLATEDB=${1:-pgstac_test_db_template} +psql -X -q -v ON_ERROR_STOP=1 <"$TMPFILE" + +diff -y --suppress-common-lines -Z -b -w -B --strip-trailing-cr "$TMPFILE" $SQLOUTFILE && echo "TEST $SQLFILE PASSED" || { echo "***TEST FOR $SQLFILE FAILED***"; exit 1; } + +done +psql -X -q -c "DROP DATABASE pgstac_test_basicsql WITH (force)"; +} - echo "Congratulations! All pypgstac tests pass." +function test_pypgstac(){ +TEMPLATEDB=${1:-pgstac_test_db_template} + cd /opt/src/pypgstac + psql -X -q -v ON_ERROR_STOP=1 < 0 ]]; do case $1 in - --skipincremental) - SKIPINCREMENTAL=1 - shift - ;; - *) - usage "Unknown parameter passed: $1" - shift - shift - ;; - esac; done - -function bail() { - echo $1 - exit 1 -} - -source $(dirname $0)/migra_funcs - -if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - echo "Running Basic SQL tests." - pushd $(dirname $0)/../../test/basic - for f in *.sql; do - echo "Running tests $f" - ./sqltest.sh $f - done - popd - - echo "Creating Temp DBs to check base and incremental migrations." - create_migra_dbs - trap drop_migra_dbs 0 2 3 15 - - echo "Using latest base migrations on Temp DB 2 $TODBURL" - pypgstac migrate --dsn $TODBURL - echo "Running PGTap Tests on Temp DB 2" - pgtap $TODBURL - - - - - if [[ ! "${SKIPINCREMENTAL}" ]]; then - echo "-----Testing incremental migrations.-----" - echo "Setting Temp DB 1 to initial version 0.1.9" - pypgstac migrate --dsn $FROMDBURL --toversion 0.1.9 - echo "Using incremental migrations to bring Temp DB 1 up to date" - pypgstac migrate --dsn $FROMDBURL - - echo "Running Basic SQL tests." - pushd $(dirname $0)/../../test/basic - for f in *.sql; do - echo "Running tests $f" - ./sqltest.sh $f - done - popd - - echo "Running PGTap Tests on Temp DB 1" - pgtap $FROMDBURL - - echo "Using Migra to test for any differences between incrementally migrated DB and DB created with latest base migration." - diffs=$(migra --schema pgstac --unsafe $FROMDBURL $TODBURL || echo "") - [ $(echo $diffs | wc -l) -gt 1 ] && { echo "DIFFERENCES FOUND: \n$diffs"; exit 1; } - fi - - - - - - echo "No differences found." - - -fi diff --git a/scripts/bin/tmpdb b/scripts/bin/tmpdb new file mode 100755 index 00000000..7bd473d0 --- /dev/null +++ b/scripts/bin/tmpdb @@ -0,0 +1,57 @@ +#!/bin/bash +set -e + +if [[ "${CI}" ]]; then + set -x +fi + +if [ ! $PGSTACDOCKER == 1 ]; then + echo "This script should only be run within pgstac docker"; exit 1; +fi + +function setuptestdb(){ + + +# Run tests using postgres user and trust authentication + +export PGDATA=/tmp/pgdata +export PGLOG=/tmp/pglog +export POSTGRES_HOST_AUTH_MODE=trust +export POSTGRES_USER=postgres +export POSTGRES_DB=postgres +export PGDATABASE=postgres +export PGUSER=postgres +export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" +export PGPORT=5438 +export POSTGRES_PORT=5438 +export POSTGRES_DATABASE=postgres + +# Make sure there is no database running from /tmp/pgdata +gosu postgres pg_ctl stop || echo "..." + +# Make sure PGDATA directory is set up from scratch +rm -fr /tmp/pgdata/* + +# Leverage scripts from docker-entrypoint.sh +source /usr/local/bin/docker-entrypoint.sh + +gosu postgres initdb --username $POSTGRES_USER --auth=trust -D $PGDATA + +# Start postgres with minimal logging settings +gosu postgres pg_ctl -l "$PGLOG" -w -o "-F -c fsync=off -c full_page_writes=off -c synchronous_commit=off -c archive_mode=off" start +trap "gosu postgres pg_ctl stop && rm -fr /tmp/pgdata/* " 0 2 3 15 + +# Get location of script directory +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +cd /opt/src/pgstac +# Create template database with pgstac installed +psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) +cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then @@ -14,7 +15,6 @@ CI build for this project. } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - scripts/install scripts/setup scripts/test fi diff --git a/scripts/cipublish b/scripts/cipublish index 25a0b885..9556826e 100755 --- a/scripts/cipublish +++ b/scripts/cipublish @@ -1,5 +1,6 @@ #!/bin/bash - +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. set -e if [[ -n "${CI}" ]]; then diff --git a/scripts/console b/scripts/console index 6224bfba..84e3a1d2 100755 --- a/scripts/console +++ b/scripts/console @@ -1,5 +1,6 @@ #!/bin/bash - +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then @@ -29,22 +30,15 @@ while [[ "$#" > 0 ]]; do case $1 in if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + docker-compose up -d + if [[ "${DB_CONSOLE}" ]]; then - scripts/server --detach - docker-compose \ - -f docker-compose.yml \ - run --rm \ - database \ - psql postgres://username:password@database:5432/postgis + docker-compose exec pgstac psql exit 0 fi - # Run database migrations - docker-compose \ - -f docker-compose.yml \ - run --rm dev \ - /bin/bash + docker-compose exec pgstac /bin/bash fi diff --git a/scripts/format b/scripts/format index 75afec38..5c687331 100755 --- a/scripts/format +++ b/scripts/format @@ -1,5 +1,6 @@ #!/bin/bash - +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then @@ -15,7 +16,6 @@ Format code in this project } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - docker-compose \ - run --rm \ - dev scripts/bin/format; + echo "Formatting pypgstac..." + docker-compose run --rm pgstac sh -c "ruff --fix pypgstac/pypgstac; ruff --fix pypgstac/tests" fi diff --git a/scripts/install b/scripts/install deleted file mode 100755 index d5e988d3..00000000 --- a/scripts/install +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -e - -if [[ "${CI}" ]]; then - set -x -fi - -function usage() { - echo -n \ - "Usage: $(basename "$0") -Installs pypgstac locally. - -Creates a local pypgstac.egg-info, which is required -even for docker development. -This is because the entire source tree gets mounted -into the container, and without a local egg-info pypgstac -will not be found inside the docker environment. -" -} - -if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - pushd pypgstac - pip install -e . - popd -fi diff --git a/scripts/migrate b/scripts/migrate index c399221d..f166d7d6 100755 --- a/scripts/migrate +++ b/scripts/migrate @@ -16,9 +16,9 @@ Run migrations against the development database. if [ "${BASH_SOURCE[0]}" = "${0}" ]; then # Run database migrations + docker-compose up -d pgstac docker-compose \ - -f docker-compose.yml \ - run --rm dev \ + exec pgstac \ bash -c "pypgstac pgready && pypgstac migrate --debug" fi diff --git a/scripts/server b/scripts/server index 9e6c8bd9..49d47192 100755 --- a/scripts/server +++ b/scripts/server @@ -1,5 +1,6 @@ #!/bin/bash - +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then @@ -33,7 +34,6 @@ while [[ "$#" > 0 ]]; do case $1 in esac; done if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - docker-compose \ - -f docker-compose.yml \ - up ${DETACH_ARG} $@ + docker-compose build + docker-compose up ${DETACH_ARG} $@ fi diff --git a/scripts/setup b/scripts/setup index 2ef9343b..c87aca3a 100755 --- a/scripts/setup +++ b/scripts/setup @@ -1,5 +1,6 @@ #!/bin/bash - +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then @@ -14,46 +15,13 @@ Sets up this project for development. } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - # Check for local install of pypgstac - if [[ ! -d "./pypgstac/pypgstac.egg-info" ]]; then - - IN_VIRTUAL_ENV=$(python -c "import sys; print(sys.prefix != sys.base_prefix)") - if [[ "${IN_VIRTUAL_ENV}" == "True" ]]; then - echo "Installing pypgstac into virtual environment..." - scripts/install - else - echo "ERROR: You must be in a virtual environment to run scripts/setup." - echo "Otherwise install pypgstac locally with scripts/install before running this script." - exit 1; - fi - fi # Build docker containers scripts/update - echo "migrating..." - scripts/migrate - echo "Bringing up database..." scripts/server --detach - echo "Ingesting development data..." - docker-compose \ - -f docker-compose.yml \ - run --rm \ - dev \ - pypgstac load collections \ - /opt/src/test/testdata/collections.ndjson \ - --method upsert - - docker-compose \ - -f docker-compose.yml \ - run --rm \ - dev \ - pypgstac load items \ - /opt/src/test/testdata/items.ndjson \ - --method upsert - echo "Done." fi diff --git a/scripts/sql/resetdb.sql b/scripts/sql/resetdb.sql deleted file mode 100644 index 4e09e418..00000000 --- a/scripts/sql/resetdb.sql +++ /dev/null @@ -1,7 +0,0 @@ -BEGIN; - DROP SCHEMA IF EXISTS pgstac CASCADE; - ALTER DATABASE postgis SET SEARCH_PATH TO pgstac, public; - \i pgstac.sql - \copy collections (content) FROM 'test/testdata/collections.ndjson' - \copy items_staging_ignore (content) FROM 'test/testdata/items.ndjson' -COMMIT; diff --git a/scripts/stageversion b/scripts/stageversion index ee15bc0b..bd0b999f 100755 --- a/scripts/stageversion +++ b/scripts/stageversion @@ -1,5 +1,6 @@ #!/bin/bash - +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then @@ -24,13 +25,9 @@ if [[ -z "${VERSION}" ]]; then exit 1 fi -cat < $(dirname $0)/../pypgstac/pypgstac/version.py -"""Version.""" -__version__ = "${VERSION}" -EOD - if [ "${BASH_SOURCE[0]}" = "${0}" ]; then echo "Updating Version..." - docker-compose run --rm dev scripts/bin/stageversion ${VERSION} - docker-compose exec -T database scripts/bin/stageversiondb ${FORCE} + docker-compose build + docker-compose run --rm pgstac stageversion $VERSION $FORCE + sudo chown -R $USER:$USER src/pgstac/migrations fi diff --git a/scripts/test b/scripts/test index 579d2747..a9d21ed4 100755 --- a/scripts/test +++ b/scripts/test @@ -1,5 +1,6 @@ #!/bin/bash - +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then @@ -8,39 +9,13 @@ fi function usage() { echo -n \ - "Usage: $(basename "$0") [--skipincremental] -Run database tests - ---skipincremental: Skip incremental migration checks. + "Usage: $(basename "$0") " } -SKIPINCREMENTAL='' -while [[ "$#" > 0 ]]; do case $1 in - --skipincremental) - SKIPINCREMENTAL='--skipincremental' - shift - ;; - *) - usage "Unknown parameter passed: $1" - shift - shift - ;; - esac; done if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - echo "Starting database..." - scripts/server --detach; - - scripts/migrate - - echo "Running database tests..." - docker-compose \ - exec -T \ - database /opt/src/scripts/bin/testdb $SKIPINCREMENTAL; - - echo "Running pypgstac tests..." - docker-compose \ - run --rm \ - dev scripts/bin/test; + docker-compose build + echo "test $@" + docker-compose run --rm pgstac test $@ fi diff --git a/scripts/update b/scripts/update index 1b5848d6..aacd3ab8 100755 --- a/scripts/update +++ b/scripts/update @@ -1,5 +1,6 @@ #!/bin/bash - +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then @@ -26,8 +27,7 @@ esac; done if [ "${BASH_SOURCE[0]}" = "${0}" ]; then echo "==Building images..." - docker-compose \ - -f docker-compose.yml \ - build ${NO_CACHE} + + docker-compose build ${NO_CACHE} fi diff --git a/sql/002_collections.sql b/sql/002_collections.sql deleted file mode 100644 index cebd42d9..00000000 --- a/sql/002_collections.sql +++ /dev/null @@ -1,496 +0,0 @@ - - - -CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ - SELECT jsonb_build_object( - 'type', 'Feature', - 'stac_version', content->'stac_version', - 'assets', content->'item_assets', - 'collection', content->'id' - ); -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - -CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); - -CREATE TABLE IF NOT EXISTS collections ( - key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, - content JSONB NOT NULL, - base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, - partition_trunc partition_trunc_strategy -); - - -CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ - SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; -$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; - - - -CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ -DECLARE - retval boolean; -BEGIN - EXECUTE format($q$ - SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) - $q$, - $1 - ) INTO retval; - RETURN retval; -END; -$$ LANGUAGE PLPGSQL; - - -CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ -DECLARE - q text; - partition_name text := format('_items_%s', NEW.key); - partition_exists boolean := false; - partition_empty boolean := true; - err_context text; - loadtemp boolean := FALSE; -BEGIN - RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; - SELECT relid::text INTO partition_name - FROM pg_partition_tree('items') - WHERE relid::text = partition_name; - IF FOUND THEN - partition_exists := true; - partition_empty := table_empty(partition_name); - ELSE - partition_exists := false; - partition_empty := true; - partition_name := format('_items_%s', NEW.key); - END IF; - IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN - q := format($q$ - DROP TABLE IF EXISTS %I CASCADE; - $q$, - partition_name - ); - EXECUTE q; - END IF; - IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN - q := format($q$ - CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; - DROP TABLE IF EXISTS %I CASCADE; - $q$, - partition_name, - partition_name - ); - EXECUTE q; - loadtemp := TRUE; - partition_empty := TRUE; - partition_exists := FALSE; - END IF; - IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN - RETURN NEW; - END IF; - IF NEW.partition_trunc IS NULL AND partition_empty THEN - RAISE NOTICE '% % % %', - partition_name, - NEW.id, - concat(partition_name,'_id_idx'), - partition_name - ; - q := format($q$ - CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); - CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); - $q$, - partition_name, - NEW.id, - concat(partition_name,'_id_idx'), - partition_name - ); - RAISE NOTICE 'q: %', q; - BEGIN - EXECUTE q; - EXCEPTION - WHEN duplicate_table THEN - RAISE NOTICE 'Partition % already exists.', partition_name; - WHEN others THEN - GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; - RAISE INFO 'Error Name:%',SQLERRM; - RAISE INFO 'Error State:%', SQLSTATE; - RAISE INFO 'Error Context:%', err_context; - END; - - ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; - DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; - ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; - - INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); - ELSIF partition_empty THEN - q := format($q$ - CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) - PARTITION BY RANGE (datetime); - $q$, - partition_name, - NEW.id - ); - RAISE NOTICE 'q: %', q; - BEGIN - EXECUTE q; - EXCEPTION - WHEN duplicate_table THEN - RAISE NOTICE 'Partition % already exists.', partition_name; - WHEN others THEN - GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; - RAISE INFO 'Error Name:%',SQLERRM; - RAISE INFO 'Error State:%', SQLSTATE; - RAISE INFO 'Error Context:%', err_context; - END; - ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; - DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; - ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; - ELSE - RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; - END IF; - IF loadtemp THEN - RAISE NOTICE 'Moving data into new partitions.'; - q := format($q$ - WITH p AS ( - SELECT - collection, - datetime as datetime, - end_datetime as end_datetime, - (partition_name( - collection, - datetime - )).partition_name as name - FROM changepartitionstaging - ) - INSERT INTO partitions (collection, datetime_range, end_datetime_range) - SELECT - collection, - tstzrange(min(datetime), max(datetime), '[]') as datetime_range, - tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range - FROM p - GROUP BY collection, name - ON CONFLICT (name) DO UPDATE SET - datetime_range = EXCLUDED.datetime_range, - end_datetime_range = EXCLUDED.end_datetime_range - ; - INSERT INTO %I SELECT * FROM changepartitionstaging; - DROP TABLE IF EXISTS changepartitionstaging; - $q$, - partition_name - ); - EXECUTE q; - END IF; - RETURN NEW; -END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; - -CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW -EXECUTE FUNCTION collections_trigger_func(); - -CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ - UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; -$$ LANGUAGE SQL; - -CREATE TABLE IF NOT EXISTS partitions ( - collection text REFERENCES collections(id) ON DELETE CASCADE, - name text PRIMARY KEY, - partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), - datetime_range tstzrange, - end_datetime_range tstzrange, - CONSTRAINT prange EXCLUDE USING GIST ( - collection WITH =, - partition_range WITH && - ) -) WITH (FILLFACTOR=90); -CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); - -CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ -DECLARE - q text; -BEGIN - RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; - EXECUTE format($q$ - DROP TABLE IF EXISTS %I CASCADE; - $q$, - OLD.name - ); - RAISE NOTICE 'Dropped partition.'; - RETURN OLD; -END; -$$ LANGUAGE PLPGSQL; - -CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW -EXECUTE FUNCTION partitions_delete_trigger_func(); - -CREATE OR REPLACE FUNCTION partition_name( - IN collection text, - IN dt timestamptz, - OUT partition_name text, - OUT partition_range tstzrange -) AS $$ -DECLARE - c RECORD; - parent_name text; -BEGIN - SELECT * INTO c FROM pgstac.collections WHERE id=collection; - IF NOT FOUND THEN - RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; - END IF; - parent_name := format('_items_%s', c.key); - - - IF c.partition_trunc = 'year' THEN - partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); - ELSIF c.partition_trunc = 'month' THEN - partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); - ELSE - partition_name := parent_name; - partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); - END IF; - IF partition_range IS NULL THEN - partition_range := tstzrange( - date_trunc(c.partition_trunc::text, dt), - date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval - ); - END IF; - RETURN; - -END; -$$ LANGUAGE PLPGSQL STABLE; - - - -CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ -DECLARE - q text; - cq text; - parent_name text; - partition_trunc text; - partition_name text := NEW.name; - partition_exists boolean := false; - partition_empty boolean := true; - partition_range tstzrange; - datetime_range tstzrange; - end_datetime_range tstzrange; - err_context text; - mindt timestamptz := lower(NEW.datetime_range); - maxdt timestamptz := upper(NEW.datetime_range); - minedt timestamptz := lower(NEW.end_datetime_range); - maxedt timestamptz := upper(NEW.end_datetime_range); - t_mindt timestamptz; - t_maxdt timestamptz; - t_minedt timestamptz; - t_maxedt timestamptz; -BEGIN - RAISE NOTICE 'Partitions Trigger. %', NEW; - datetime_range := NEW.datetime_range; - end_datetime_range := NEW.end_datetime_range; - - SELECT - format('_items_%s', key), - c.partition_trunc::text - INTO - parent_name, - partition_trunc - FROM pgstac.collections c - WHERE c.id = NEW.collection; - SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; - NEW.name := partition_name; - - IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN - partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); - END IF; - - NEW.partition_range := partition_range; - IF TG_OP = 'UPDATE' THEN - mindt := least(mindt, lower(OLD.datetime_range)); - maxdt := greatest(maxdt, upper(OLD.datetime_range)); - minedt := least(minedt, lower(OLD.end_datetime_range)); - maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); - NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); - NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); - END IF; - IF TG_OP = 'INSERT' THEN - - IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN - - RAISE NOTICE '% % %', partition_name, parent_name, partition_range; - q := format($q$ - CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); - CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); - $q$, - partition_name, - parent_name, - lower(partition_range), - upper(partition_range), - format('%s_pkey', partition_name), - partition_name - ); - BEGIN - EXECUTE q; - EXCEPTION - WHEN duplicate_table THEN - RAISE NOTICE 'Partition % already exists.', partition_name; - WHEN others THEN - GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; - RAISE INFO 'Error Name:%',SQLERRM; - RAISE INFO 'Error State:%', SQLSTATE; - RAISE INFO 'Error Context:%', err_context; - END; - END IF; - - END IF; - - -- Update constraints - EXECUTE format($q$ - SELECT - min(datetime), - max(datetime), - min(end_datetime), - max(end_datetime) - FROM %I; - $q$, partition_name) - INTO t_mindt, t_maxdt, t_minedt, t_maxedt; - mindt := least(mindt, t_mindt); - maxdt := greatest(maxdt, t_maxdt); - minedt := least(mindt, minedt, t_minedt); - maxedt := greatest(maxdt, maxedt, t_maxedt); - - mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); - maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; - minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); - maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; - - - IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN - NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); - NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); - IF - TG_OP='UPDATE' - AND OLD.datetime_range @> NEW.datetime_range - AND OLD.end_datetime_range @> NEW.end_datetime_range - THEN - RAISE NOTICE 'Range unchanged, not updating constraints.'; - ELSE - - RAISE NOTICE ' - SETTING CONSTRAINTS - mindt: %, maxdt: % - minedt: %, maxedt: % - ', mindt, maxdt, minedt, maxedt; - IF partition_trunc IS NULL THEN - cq := format($q$ - ALTER TABLE %7$I - DROP CONSTRAINT IF EXISTS %1$I, - DROP CONSTRAINT IF EXISTS %2$I, - ADD CONSTRAINT %1$I - CHECK ( - (datetime >= %3$L) - AND (datetime <= %4$L) - AND (end_datetime >= %5$L) - AND (end_datetime <= %6$L) - ) NOT VALID - ; - ALTER TABLE %7$I - VALIDATE CONSTRAINT %1$I; - $q$, - format('%s_dt', partition_name), - format('%s_edt', partition_name), - mindt, - maxdt, - minedt, - maxedt, - partition_name - ); - ELSE - cq := format($q$ - ALTER TABLE %5$I - DROP CONSTRAINT IF EXISTS %1$I, - DROP CONSTRAINT IF EXISTS %2$I, - ADD CONSTRAINT %2$I - CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID - ; - ALTER TABLE %5$I - VALIDATE CONSTRAINT %2$I; - $q$, - format('%s_dt', partition_name), - format('%s_edt', partition_name), - minedt, - maxedt, - partition_name - ); - - END IF; - RAISE NOTICE 'Altering Constraints. %', cq; - EXECUTE cq; - END IF; - ELSE - NEW.datetime_range = NULL; - NEW.end_datetime_range = NULL; - - cq := format($q$ - ALTER TABLE %3$I - DROP CONSTRAINT IF EXISTS %1$I, - DROP CONSTRAINT IF EXISTS %2$I, - ADD CONSTRAINT %1$I - CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID - ; - ALTER TABLE %3$I - VALIDATE CONSTRAINT %1$I; - $q$, - format('%s_dt', partition_name), - format('%s_edt', partition_name), - partition_name - ); - EXECUTE cq; - END IF; - - RETURN NEW; - -END; -$$ LANGUAGE PLPGSQL; - -CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW -EXECUTE FUNCTION partitions_trigger_func(); - - -CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ - INSERT INTO collections (content) - VALUES (data) - ; -$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; - -CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ -DECLARE - out collections%ROWTYPE; -BEGIN - UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; -END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; - -CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ - INSERT INTO collections (content) - VALUES (data) - ON CONFLICT (id) DO - UPDATE - SET content=EXCLUDED.content - ; -$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; - -CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ -DECLARE - out collections%ROWTYPE; -BEGIN - DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; -END; -$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; - - -CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ - SELECT content FROM collections - WHERE id=$1 - ; -$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; - -CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ - SELECT jsonb_agg(content) FROM collections; -; -$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; diff --git a/sql/998_permissions.sql b/sql/998_permissions.sql deleted file mode 100644 index 13d8696f..00000000 --- a/sql/998_permissions.sql +++ /dev/null @@ -1,13 +0,0 @@ -GRANT USAGE ON SCHEMA pgstac to pgstac_read; -GRANT ALL ON SCHEMA pgstac to pgstac_ingest; -GRANT ALL ON SCHEMA pgstac to pgstac_admin; - --- pgstac_read role limited to using function apis -GRANT EXECUTE ON FUNCTION search TO pgstac_read; -GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; -GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; -GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; - -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; -GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; -GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; diff --git a/sql/999_version.sql b/sql/999_version.sql deleted file mode 100644 index ae7ae1d4..00000000 --- a/sql/999_version.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT set_version('0.6.13'); diff --git a/pypgstac/pypgstac/migrations/pgstac.0.1.9-0.2.3.sql b/src/pgstac/migrations/pgstac.0.1.9-0.2.3.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.1.9-0.2.3.sql rename to src/pgstac/migrations/pgstac.0.1.9-0.2.3.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.1.9.sql b/src/pgstac/migrations/pgstac.0.1.9.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.1.9.sql rename to src/pgstac/migrations/pgstac.0.1.9.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.3-0.2.4.sql b/src/pgstac/migrations/pgstac.0.2.3-0.2.4.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.3-0.2.4.sql rename to src/pgstac/migrations/pgstac.0.2.3-0.2.4.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.3.sql b/src/pgstac/migrations/pgstac.0.2.3.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.3.sql rename to src/pgstac/migrations/pgstac.0.2.3.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.4-0.2.5.sql b/src/pgstac/migrations/pgstac.0.2.4-0.2.5.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.4-0.2.5.sql rename to src/pgstac/migrations/pgstac.0.2.4-0.2.5.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.4-0.2.7.sql b/src/pgstac/migrations/pgstac.0.2.4-0.2.7.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.4-0.2.7.sql rename to src/pgstac/migrations/pgstac.0.2.4-0.2.7.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.4.sql b/src/pgstac/migrations/pgstac.0.2.4.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.4.sql rename to src/pgstac/migrations/pgstac.0.2.4.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.5-0.2.7.sql b/src/pgstac/migrations/pgstac.0.2.5-0.2.7.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.5-0.2.7.sql rename to src/pgstac/migrations/pgstac.0.2.5-0.2.7.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.5.sql b/src/pgstac/migrations/pgstac.0.2.5.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.5.sql rename to src/pgstac/migrations/pgstac.0.2.5.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.7-0.2.8.sql b/src/pgstac/migrations/pgstac.0.2.7-0.2.8.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.7-0.2.8.sql rename to src/pgstac/migrations/pgstac.0.2.7-0.2.8.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.7.sql b/src/pgstac/migrations/pgstac.0.2.7.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.7.sql rename to src/pgstac/migrations/pgstac.0.2.7.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.8-0.2.9.sql b/src/pgstac/migrations/pgstac.0.2.8-0.2.9.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.8-0.2.9.sql rename to src/pgstac/migrations/pgstac.0.2.8-0.2.9.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.8.sql b/src/pgstac/migrations/pgstac.0.2.8.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.8.sql rename to src/pgstac/migrations/pgstac.0.2.8.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.9-0.3.0.sql b/src/pgstac/migrations/pgstac.0.2.9-0.3.0.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.9-0.3.0.sql rename to src/pgstac/migrations/pgstac.0.2.9-0.3.0.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.2.9.sql b/src/pgstac/migrations/pgstac.0.2.9.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.2.9.sql rename to src/pgstac/migrations/pgstac.0.2.9.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.0-0.3.1.sql b/src/pgstac/migrations/pgstac.0.3.0-0.3.1.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.0-0.3.1.sql rename to src/pgstac/migrations/pgstac.0.3.0-0.3.1.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.0.sql b/src/pgstac/migrations/pgstac.0.3.0.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.0.sql rename to src/pgstac/migrations/pgstac.0.3.0.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.1-0.3.2.sql b/src/pgstac/migrations/pgstac.0.3.1-0.3.2.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.1-0.3.2.sql rename to src/pgstac/migrations/pgstac.0.3.1-0.3.2.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.1.sql b/src/pgstac/migrations/pgstac.0.3.1.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.1.sql rename to src/pgstac/migrations/pgstac.0.3.1.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.2-0.3.3.sql b/src/pgstac/migrations/pgstac.0.3.2-0.3.3.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.2-0.3.3.sql rename to src/pgstac/migrations/pgstac.0.3.2-0.3.3.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.2.sql b/src/pgstac/migrations/pgstac.0.3.2.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.2.sql rename to src/pgstac/migrations/pgstac.0.3.2.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.3-0.3.4.sql b/src/pgstac/migrations/pgstac.0.3.3-0.3.4.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.3-0.3.4.sql rename to src/pgstac/migrations/pgstac.0.3.3-0.3.4.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.3.sql b/src/pgstac/migrations/pgstac.0.3.3.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.3.sql rename to src/pgstac/migrations/pgstac.0.3.3.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.4-0.3.5.sql b/src/pgstac/migrations/pgstac.0.3.4-0.3.5.sql similarity index 99% rename from pypgstac/pypgstac/migrations/pgstac.0.3.4-0.3.5.sql rename to src/pgstac/migrations/pgstac.0.3.4-0.3.5.sql index b0450a77..7dff28a0 100644 --- a/pypgstac/pypgstac/migrations/pgstac.0.3.4-0.3.5.sql +++ b/src/pgstac/migrations/pgstac.0.3.4-0.3.5.sql @@ -6,7 +6,7 @@ INSERT INTO pgstac.migrations SELECT * FROM temp_migrations; drop function if exists "pgstac"."partition_queries"(_where text, _orderby text); -drop function if exists "pgstac"."search_hash"(jsonb); +--drop function if exists "pgstac"."search_hash"(jsonb); drop function if exists "pgstac"."search_query"(_search jsonb, updatestats boolean); @@ -52,6 +52,7 @@ AS $function$ $function$ ; +alter table "pgstac"."searches" DROP COLUMN hash; alter table "pgstac"."searches" add column "hash" text generated always as ("pgstac".search_hash(search, metadata)) stored primary key; CREATE UNIQUE INDEX migrations_pkey ON pgstac.migrations USING btree (version); diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.4.sql b/src/pgstac/migrations/pgstac.0.3.4.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.4.sql rename to src/pgstac/migrations/pgstac.0.3.4.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.5-0.3.6.sql b/src/pgstac/migrations/pgstac.0.3.5-0.3.6.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.5-0.3.6.sql rename to src/pgstac/migrations/pgstac.0.3.5-0.3.6.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.5.sql b/src/pgstac/migrations/pgstac.0.3.5.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.5.sql rename to src/pgstac/migrations/pgstac.0.3.5.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.6-0.4.0.sql b/src/pgstac/migrations/pgstac.0.3.6-0.4.0.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.6-0.4.0.sql rename to src/pgstac/migrations/pgstac.0.3.6-0.4.0.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.3.6.sql b/src/pgstac/migrations/pgstac.0.3.6.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.3.6.sql rename to src/pgstac/migrations/pgstac.0.3.6.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.0-0.4.1.sql b/src/pgstac/migrations/pgstac.0.4.0-0.4.1.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.0-0.4.1.sql rename to src/pgstac/migrations/pgstac.0.4.0-0.4.1.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.0.sql b/src/pgstac/migrations/pgstac.0.4.0.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.0.sql rename to src/pgstac/migrations/pgstac.0.4.0.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.1-0.4.2.sql b/src/pgstac/migrations/pgstac.0.4.1-0.4.2.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.1-0.4.2.sql rename to src/pgstac/migrations/pgstac.0.4.1-0.4.2.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.1.sql b/src/pgstac/migrations/pgstac.0.4.1.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.1.sql rename to src/pgstac/migrations/pgstac.0.4.1.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.2-0.4.3.sql b/src/pgstac/migrations/pgstac.0.4.2-0.4.3.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.2-0.4.3.sql rename to src/pgstac/migrations/pgstac.0.4.2-0.4.3.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.2.sql b/src/pgstac/migrations/pgstac.0.4.2.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.2.sql rename to src/pgstac/migrations/pgstac.0.4.2.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.3-0.4.4.sql b/src/pgstac/migrations/pgstac.0.4.3-0.4.4.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.3-0.4.4.sql rename to src/pgstac/migrations/pgstac.0.4.3-0.4.4.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.3.sql b/src/pgstac/migrations/pgstac.0.4.3.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.3.sql rename to src/pgstac/migrations/pgstac.0.4.3.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.4-0.4.5.sql b/src/pgstac/migrations/pgstac.0.4.4-0.4.5.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.4-0.4.5.sql rename to src/pgstac/migrations/pgstac.0.4.4-0.4.5.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.4.sql b/src/pgstac/migrations/pgstac.0.4.4.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.4.sql rename to src/pgstac/migrations/pgstac.0.4.4.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.5-0.5.0.sql b/src/pgstac/migrations/pgstac.0.4.5-0.5.0.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.5-0.5.0.sql rename to src/pgstac/migrations/pgstac.0.4.5-0.5.0.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.4.5.sql b/src/pgstac/migrations/pgstac.0.4.5.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.4.5.sql rename to src/pgstac/migrations/pgstac.0.4.5.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.5.0-0.5.1.sql b/src/pgstac/migrations/pgstac.0.5.0-0.5.1.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.5.0-0.5.1.sql rename to src/pgstac/migrations/pgstac.0.5.0-0.5.1.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.5.0.sql b/src/pgstac/migrations/pgstac.0.5.0.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.5.0.sql rename to src/pgstac/migrations/pgstac.0.5.0.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.5.1-0.6.0.sql b/src/pgstac/migrations/pgstac.0.5.1-0.6.0.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.5.1-0.6.0.sql rename to src/pgstac/migrations/pgstac.0.5.1-0.6.0.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.5.1.sql b/src/pgstac/migrations/pgstac.0.5.1.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.5.1.sql rename to src/pgstac/migrations/pgstac.0.5.1.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.0-0.6.1.sql b/src/pgstac/migrations/pgstac.0.6.0-0.6.1.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.0-0.6.1.sql rename to src/pgstac/migrations/pgstac.0.6.0-0.6.1.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.0.sql b/src/pgstac/migrations/pgstac.0.6.0.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.0.sql rename to src/pgstac/migrations/pgstac.0.6.0.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.1-0.6.2.sql b/src/pgstac/migrations/pgstac.0.6.1-0.6.2.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.1-0.6.2.sql rename to src/pgstac/migrations/pgstac.0.6.1-0.6.2.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.1.sql b/src/pgstac/migrations/pgstac.0.6.1.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.1.sql rename to src/pgstac/migrations/pgstac.0.6.1.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.10-0.6.11.sql b/src/pgstac/migrations/pgstac.0.6.10-0.6.11.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.10-0.6.11.sql rename to src/pgstac/migrations/pgstac.0.6.10-0.6.11.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.10.sql b/src/pgstac/migrations/pgstac.0.6.10.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.10.sql rename to src/pgstac/migrations/pgstac.0.6.10.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.11-0.6.12.sql b/src/pgstac/migrations/pgstac.0.6.11-0.6.12.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.11-0.6.12.sql rename to src/pgstac/migrations/pgstac.0.6.11-0.6.12.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.11.sql b/src/pgstac/migrations/pgstac.0.6.11.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.11.sql rename to src/pgstac/migrations/pgstac.0.6.11.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.12-0.6.13.sql b/src/pgstac/migrations/pgstac.0.6.12-0.6.13.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.12-0.6.13.sql rename to src/pgstac/migrations/pgstac.0.6.12-0.6.13.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.12.sql b/src/pgstac/migrations/pgstac.0.6.12.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.12.sql rename to src/pgstac/migrations/pgstac.0.6.12.sql diff --git a/src/pgstac/migrations/pgstac.0.6.13-0.7.0.sql b/src/pgstac/migrations/pgstac.0.6.13-0.7.0.sql new file mode 100644 index 00000000..f4a9ae06 --- /dev/null +++ b/src/pgstac/migrations/pgstac.0.6.13-0.7.0.sql @@ -0,0 +1,1782 @@ +DO $$ +DECLARE +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN + CREATE EXTENSION IF NOT EXISTS postgis; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN + CREATE EXTENSION IF NOT EXISTS btree_gist; + END IF; +END; +$$ LANGUAGE PLPGSQL; + +DO $$ + BEGIN + CREATE ROLE pgstac_admin; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_read; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_ingest; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + + +GRANT pgstac_admin TO current_user; + +-- Function to make sure pgstac_admin is the owner of items +CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ +DECLARE + f RECORD; +BEGIN + FOR f IN ( + SELECT + concat( + oid::regproc::text, + '(', + coalesce(pg_get_function_identity_arguments(oid),''), + ')' + ) AS name, + CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ + FROM pg_proc + WHERE + pronamespace=to_regnamespace('pgstac') + AND proowner != to_regrole('pgstac_admin') + AND proname NOT LIKE 'pg_stat%' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + FOR f IN ( + SELECT + oid::regclass::text as name, + CASE relkind + WHEN 'i' THEN 'INDEX' + WHEN 'I' THEN 'INDEX' + WHEN 'p' THEN 'TABLE' + WHEN 'r' THEN 'TABLE' + WHEN 'v' THEN 'VIEW' + WHEN 'S' THEN 'SEQUENCE' + ELSE NULL + END as typ + FROM pg_class + WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + RETURN; +END; +$$ LANGUAGE PLPGSQL; +SELECT pgstac_admin_owns(); + +CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; + +GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; + +ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; + +GRANT pgstac_read TO pgstac_ingest; +GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; + +SET ROLE pgstac_admin; + +SET SEARCH_PATH TO pgstac, public; + +DROP FUNCTION IF EXISTS analyze_items; +DROP FUNCTION IF EXISTS validate_constraints; +SET client_min_messages TO WARNING; +SET SEARCH_PATH to pgstac, public; +-- BEGIN migra calculated SQL +drop trigger if exists "queryables_collection_trigger" on "pgstac"."collections"; + +drop trigger if exists "partitions_delete_trigger" on "pgstac"."partitions"; + +drop trigger if exists "partitions_trigger" on "pgstac"."partitions"; + +alter table "pgstac"."partitions" drop constraint "partitions_collection_fkey"; + +alter table "pgstac"."partitions" drop constraint "prange"; + +drop function if exists "pgstac"."create_queryable_indexes"(); + +drop function if exists "pgstac"."partition_collection"(collection text, strategy pgstac.partition_trunc_strategy); + +drop function if exists "pgstac"."partitions_delete_trigger_func"(); + +drop function if exists "pgstac"."partitions_trigger_func"(); + +drop function if exists "pgstac"."parse_dtrange"(_indate jsonb, relative_base timestamp with time zone); + +drop view if exists "pgstac"."partition_steps"; + +alter table "pgstac"."partitions" drop constraint "partitions_pkey"; + +drop index if exists "pgstac"."partitions_pkey"; + +select 1; -- drop index if exists "pgstac"."prange"; + +drop index if exists "pgstac"."partitions_range_idx"; + +drop table "pgstac"."partitions"; + +create table "pgstac"."partition_stats" ( + "partition" text not null, + "dtrange" tstzrange, + "edtrange" tstzrange, + "spatial" geometry, + "last_updated" timestamp with time zone, + "keys" text[] +); + + +create table "pgstac"."query_queue" ( + "query" text not null, + "added" timestamp with time zone default now() +); + + +create table "pgstac"."query_queue_history" ( + "query" text, + "added" timestamp with time zone not null, + "finished" timestamp with time zone not null default now(), + "error" text +); + + +alter table "pgstac"."collections" alter column "partition_trunc" set data type text using "partition_trunc"::text; + +drop type "pgstac"."partition_trunc_strategy"; + +CREATE UNIQUE INDEX partition_stats_pkey ON pgstac.partition_stats USING btree (partition); + +CREATE UNIQUE INDEX query_queue_pkey ON pgstac.query_queue USING btree (query); + +CREATE INDEX partitions_range_idx ON pgstac.partition_stats USING gist (dtrange); + +alter table "pgstac"."partition_stats" add constraint "partition_stats_pkey" PRIMARY KEY using index "partition_stats_pkey"; + +alter table "pgstac"."query_queue" add constraint "query_queue_pkey" PRIMARY KEY using index "query_queue_pkey"; + +alter table "pgstac"."collections" add constraint "collections_partition_trunc_check" CHECK ((partition_trunc = ANY (ARRAY['year'::text, 'month'::text]))) not valid; + +alter table "pgstac"."collections" validate constraint "collections_partition_trunc_check"; + +set check_function_bodies = off; + +CREATE OR REPLACE FUNCTION pgstac.check_partition(_collection text, _dtrange tstzrange, _edtrange tstzrange) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER +AS $function$ +DECLARE + c RECORD; + pm RECORD; + _partition_name text; + _partition_dtrange tstzrange; + _constraint_dtrange tstzrange; + _constraint_edtrange tstzrange; + q text; + deferrable_q text; + err_context text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + + IF c.partition_trunc IS NOT NULL THEN + _partition_dtrange := tstzrange( + date_trunc(c.partition_trunc, lower(_dtrange)), + date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, + '[)' + ); + ELSE + _partition_dtrange := '[-infinity, infinity]'::tstzrange; + END IF; + + IF NOT _partition_dtrange @> _dtrange THEN + RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; + END IF; + + + IF c.partition_trunc = 'year' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); + ELSE + _partition_name := format('_items_%s', c.key); + END IF; + + SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; + IF FOUND THEN + RAISE NOTICE '% % %', _edtrange, _dtrange, pm; + _constraint_edtrange := + tstzrange( + least( + lower(_edtrange), + nullif(lower(pm.constraint_edtrange), '-infinity') + ), + greatest( + upper(_edtrange), + nullif(upper(pm.constraint_edtrange), 'infinity') + ), + '[]' + ); + _constraint_dtrange := + tstzrange( + least( + lower(_dtrange), + nullif(lower(pm.constraint_dtrange), '-infinity') + ), + greatest( + upper(_dtrange), + nullif(upper(pm.constraint_dtrange), 'infinity') + ), + '[]' + ); + + IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN + RETURN pm.partition; + ELSE + PERFORM drop_table_constraints(_partition_name); + END IF; + ELSE + _constraint_edtrange := _edtrange; + _constraint_dtrange := _dtrange; + END IF; + RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; + IF c.partition_trunc IS NULL THEN + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + _partition_name, + _collection, + concat(_partition_name,'_pk'), + _partition_name + ); + ELSE + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); + CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + format('_items_%s', c.key), + _collection, + _partition_name, + format('_items_%s', c.key), + lower(_partition_dtrange), + upper(_partition_dtrange), + format('%s_pk', _partition_name), + _partition_name + ); + END IF; + + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', _partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); + PERFORM maintain_partitions(_partition_name); + PERFORM update_partition_stats_q(_partition_name, true); + RETURN _partition_name; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.collection_extent(_collection text, runupdate boolean DEFAULT false) + RETURNS jsonb + LANGUAGE plpgsql +AS $function$ +DECLARE + geom_extent geometry; + mind timestamptz; + maxd timestamptz; + extent jsonb; +BEGIN + IF runupdate THEN + PERFORM update_partition_stats_q(partition) + FROM partitions WHERE collection=_collection; + END IF; + SELECT + min(lower(dtrange)), + max(upper(edtrange)), + st_extent(spatial) + INTO + mind, + maxd, + geom_extent + FROM partitions + WHERE collection=_collection; + + IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN + extent := jsonb_build_object( + 'extent', jsonb_build_object( + 'spatial', jsonb_build_object( + 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) + ), + 'temporal', jsonb_build_object( + 'interval', to_jsonb(array[array[mind, maxd]]) + ) + ) + ); + RETURN extent; + END IF; + RETURN NULL; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.constraint_tstzrange(expr text) + RETURNS tstzrange + LANGUAGE sql + IMMUTABLE PARALLEL SAFE STRICT +AS $function$ + WITH t AS ( + SELECT regexp_matches( + expr, + E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' + ) AS m + ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t + ; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER +AS $function$ +DECLARE + q text; +BEGIN + IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN + RETURN NULL; + END IF; + RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; + IF _dtrange = 'empty' AND _edtrange = 'empty' THEN + q :=format( + $q$ + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + $q$, + t, + format('%s_dt', t), + t, + format('%s_dt', t) + ); + ELSE + q :=format( + $q$ + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK ( + (datetime >= %L) + AND (datetime <= %L) + AND (end_datetime >= %L) + AND (end_datetime <= %L) + ) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + $q$, + t, + format('%s_dt', t), + lower(_dtrange), + upper(_dtrange), + lower(_edtrange), + upper(_edtrange), + t, + format('%s_dt', t) + ); + END IF; + PERFORM run_or_queue(q); + RETURN t; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.drop_table_constraints(t text) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER +AS $function$ +DECLARE + q text; +BEGIN + IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN + RETURN NULL; + END IF; + FOR q IN SELECT FORMAT( + $q$ + ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; + $q$, + t, + conname + ) FROM pg_constraint + WHERE conrelid=t::regclass::oid AND contype='c' + LOOP + EXECUTE q; + END LOOP; + RETURN t; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) + RETURNS record + LANGUAGE plpgsql + STABLE STRICT +AS $function$ +DECLARE + expr text := pg_get_constraintdef(coid); + matches timestamptz[]; +BEGIN + IF expr LIKE '%NULL%' THEN + dt := tstzrange(null::timestamptz, null::timestamptz); + edt := tstzrange(null::timestamptz, null::timestamptz); + RETURN; + END IF; + WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) + SELECT array_agg(f::timestamptz) INTO matches FROM f; + IF cardinality(matches) = 4 THEN + dt := tstzrange(matches[1], matches[2],'[]'); + edt := tstzrange(matches[3], matches[4], '[]'); + RETURN; + ELSIF cardinality(matches) = 2 THEN + edt := tstzrange(matches[1], matches[2],'[]'); + RETURN; + END IF; + RETURN; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.get_setting_bool(_setting text, conf jsonb DEFAULT NULL::jsonb) + RETURNS boolean + LANGUAGE sql +AS $function$ +SELECT COALESCE( + conf->>_setting, + current_setting(concat('pgstac.',_setting), TRUE), + (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), + 'FALSE' +)::boolean; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false) + RETURNS SETOF text + LANGUAGE plpgsql +AS $function$ +DECLARE + parent text; + level int; + isleaf bool; + collection collections%ROWTYPE; + subpart text; + baseidx text; + queryable_name text; + queryable_property_index_type text; + queryable_property_wrapper text; + queryable_parsed RECORD; + deletedidx pg_indexes%ROWTYPE; + q text; + idx text; + collection_partition bigint; + _concurrently text := ''; +BEGIN + RAISE NOTICE 'Maintaining partition: %', part; + IF get_setting_bool('use_queue') THEN + _concurrently='CONCURRENTLY'; + END IF; + + -- Get root partition + SELECT parentrelid::text, pt.isleaf, pt.level + INTO parent, isleaf, level + FROM pg_partition_tree('items') pt + WHERE relid::text = part; + IF NOT FOUND THEN + RAISE NOTICE 'Partition % Does Not Exist In Partition Tree', part; + RETURN; + END IF; + + -- If this is a parent partition, recurse to leaves + IF NOT isleaf THEN + FOR subpart IN + SELECT relid::text + FROM pg_partition_tree(part) + WHERE relid::text != part + LOOP + RAISE NOTICE 'Recursing to %', subpart; + RETURN QUERY SELECT * FROM maintain_partition_queries(subpart, dropindexes, rebuildindexes); + END LOOP; + RETURN; -- Don't continue since not an end leaf + END IF; + + + -- Get collection + collection_partition := ((regexp_match(part, E'^_items_([0-9]+)'))[1])::bigint; + RAISE NOTICE 'COLLECTION PARTITION: %', collection_partition; + SELECT * INTO STRICT collection + FROM collections + WHERE key = collection_partition; + RAISE NOTICE 'COLLECTION ID: %s', collection.id; + + + -- Create temp table with existing indexes + CREATE TEMP TABLE existing_indexes ON COMMIT DROP AS + SELECT * + FROM pg_indexes + WHERE schemaname='pgstac' AND tablename=part; + + + -- Check if index exists for each queryable. + FOR + queryable_name, + queryable_property_index_type, + queryable_property_wrapper + IN + SELECT + name, + COALESCE(property_index_type, 'BTREE'), + COALESCE(property_wrapper, 'to_text') + FROM queryables + WHERE + name NOT in ('id', 'datetime', 'geometry') + AND ( + collection_ids IS NULL + OR collection_ids = '{}'::text[] + OR collection.id = ANY (collection_ids) + ) + UNION ALL + SELECT 'datetime desc, end_datetime', 'BTREE', '' + UNION ALL + SELECT 'geometry', 'GIST', '' + UNION ALL + SELECT 'id', 'BTREE', '' + LOOP + baseidx := format( + $q$ ON %I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, + part, + queryable_property_index_type, + queryable_property_wrapper, + queryable_name + ); + RAISE NOTICE 'BASEIDX: %', baseidx; + RAISE NOTICE 'IDXSEARCH: %', format($q$[(']%s[')]$q$, queryable_name); + -- If index already exists, delete it from existing indexes type table + FOR deletedidx IN + DELETE FROM existing_indexes + WHERE indexdef ~* format($q$[(']%s[')]$q$, queryable_name) + RETURNING * + LOOP + RAISE NOTICE 'EXISTING INDEX: %', deletedidx; + IF NOT FOUND THEN -- index did not exist, create it + RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); + ELSIF rebuildindexes THEN + RETURN NEXT format('REINDEX %I %s;', deletedidx.indexname, _concurrently); + END IF; + END LOOP; + END LOOP; + + -- Remove indexes that were not expected + FOR idx IN SELECT indexname::text FROM existing_indexes + LOOP + RAISE WARNING 'Index: % is not defined by queryables.', idx; + IF dropindexes THEN + RETURN NEXT format('DROP INDEX IF EXISTS %I;', idx); + END IF; + END LOOP; + + DROP TABLE existing_indexes; + RAISE NOTICE 'Returning from maintain_partition_queries.'; + RETURN; + +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.maintain_partitions(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false) + RETURNS void + LANGUAGE sql +AS $function$ + WITH t AS ( + SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q + ) SELECT count(*) FROM t; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.partition_after_triggerfunc() + RETURNS trigger + LANGUAGE plpgsql + SECURITY DEFINER +AS $function$ +DECLARE + p text; + t timestamptz := clock_timestamp(); +BEGIN + RAISE NOTICE 'Updating partition stats %', t; + FOR p IN SELECT DISTINCT partition + FROM newdata n JOIN partition_sys_meta p + ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) + LOOP + PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); + END LOOP; + RAISE NOTICE 't: % %', t, clock_timestamp() - t; + RETURN NULL; +END; +$function$ +; + +create or replace view "pgstac"."partition_sys_meta" as SELECT (pg_partition_tree.relid)::text AS partition, + replace(replace( + CASE + WHEN (pg_partition_tree.level = 1) THEN pg_get_expr(c.relpartbound, c.oid) + ELSE pg_get_expr(parent.relpartbound, parent.oid) + END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, + pg_partition_tree.level, + c.reltuples, + c.relhastriggers, + COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS partition_dtrange, + COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_dtrange, + COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_edtrange + FROM (((pg_partition_tree('pgstac.items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) + JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) + JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) + LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) + WHERE pg_partition_tree.isleaf; + + +create or replace view "pgstac"."partitions" as SELECT partition_sys_meta.partition, + partition_sys_meta.collection, + partition_sys_meta.level, + partition_sys_meta.reltuples, + partition_sys_meta.relhastriggers, + partition_sys_meta.partition_dtrange, + partition_sys_meta.constraint_dtrange, + partition_sys_meta.constraint_edtrange, + partition_stats.dtrange, + partition_stats.edtrange, + partition_stats.spatial, + partition_stats.last_updated, + partition_stats.keys + FROM (pgstac.partition_sys_meta + LEFT JOIN pgstac.partition_stats USING (partition)); + + +create or replace view "pgstac"."pgstac_indexes" as SELECT i.schemaname, + i.tablename, + i.indexname, + i.indexdef, + COALESCE((regexp_match(i.indexdef, '\(([a-zA-Z]+)\)'::text))[1], (regexp_match(i.indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'::text))[1], + CASE + WHEN (i.indexdef ~* '\(datetime desc, end_datetime\)'::text) THEN 'datetime_end_datetime'::text + ELSE NULL::text + END) AS field, + pg_table_size(((i.indexname)::text)::regclass) AS index_size, + pg_size_pretty(pg_table_size(((i.indexname)::text)::regclass)) AS index_size_pretty + FROM pg_indexes i + WHERE ((i.schemaname = 'pgstac'::name) AND (i.tablename ~ '_items_'::text)); + + +create or replace view "pgstac"."pgstac_indexes_stats" as SELECT i.schemaname, + i.tablename, + i.indexname, + i.indexdef, + COALESCE((regexp_match(i.indexdef, '\(([a-zA-Z]+)\)'::text))[1], (regexp_match(i.indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'::text))[1], + CASE + WHEN (i.indexdef ~* '\(datetime desc, end_datetime\)'::text) THEN 'datetime_end_datetime'::text + ELSE NULL::text + END) AS field, + pg_table_size(((i.indexname)::text)::regclass) AS index_size, + pg_size_pretty(pg_table_size(((i.indexname)::text)::regclass)) AS index_size_pretty, + s.n_distinct, + ((s.most_common_vals)::text)::text[] AS most_common_vals, + ((s.most_common_freqs)::text)::text[] AS most_common_freqs, + ((s.histogram_bounds)::text)::text[] AS histogram_bounds, + s.correlation + FROM (pg_indexes i + LEFT JOIN pg_stats s ON ((s.tablename = i.indexname))) + WHERE ((i.schemaname = 'pgstac'::name) AND (i.tablename ~ '_items_'::text)); + + +CREATE OR REPLACE FUNCTION pgstac.queue_timeout() + RETURNS interval + LANGUAGE sql +AS $function$ + SELECT set_config( + 'statement_timeout', + t2s(coalesce( + get_setting('queue_timeout'), + '1h' + )), + false + )::interval; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT false) + RETURNS text + LANGUAGE plpgsql + SECURITY DEFINER +AS $function$ +DECLARE + c RECORD; + q text; + from_trunc text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + IF triggered THEN + RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; + ELSE + RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; + IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN + RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; + RETURN _collection; + END IF; + END IF; + + IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN + EXECUTE format( + $q$ + CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; + DROP TABLE IF EXISTS %I CASCADE; + WITH p AS ( + SELECT + collection, + CASE WHEN %L IS NULL THEN '-infinity'::timestamptz + ELSE date_trunc(%L, datetime) + END as d, + tstzrange(min(datetime),max(datetime),'[]') as dtrange, + tstzrange(min(datetime),max(datetime),'[]') as edtrange + FROM changepartitionstaging + GROUP BY 1,2 + ) SELECT check_partition(collection, dtrange, edtrange) FROM p; + INSERT INTO items SELECT * FROM changepartitionstaging; + DROP TABLE changepartitionstaging; + $q$, + concat('_items_', c.key), + concat('_items_', c.key), + c.partition_trunc, + c.partition_trunc + ); + END IF; + RETURN _collection; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.run_or_queue(query text) + RETURNS void + LANGUAGE plpgsql +AS $function$ +DECLARE + use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; +BEGIN + IF get_setting_bool('debug') THEN + RAISE NOTICE '%', query; + END IF; + IF use_queue THEN + INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; + ELSE + EXECUTE query; + END IF; + RETURN; +END; +$function$ +; + +CREATE OR REPLACE PROCEDURE pgstac.run_queued_queries() + LANGUAGE plpgsql +AS $procedure$ +DECLARE + qitem query_queue%ROWTYPE; + timeout_ts timestamptz; + error text; + cnt int := 0; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; + IF NOT FOUND THEN + EXIT; + END IF; + cnt := cnt + 1; + BEGIN + RAISE NOTICE 'RUNNING QUERY: %', qitem.query; + EXECUTE qitem.query; + EXCEPTION WHEN others THEN + error := format('%s | %s', SQLERRM, SQLSTATE); + END; + INSERT INTO query_queue_history (query, added, finished, error) + VALUES (qitem.query, qitem.added, clock_timestamp(), error); + COMMIT; + END LOOP; +END; +$procedure$ +; + +CREATE OR REPLACE FUNCTION pgstac.run_queued_queries_intransaction() + RETURNS integer + LANGUAGE plpgsql +AS $function$ +DECLARE + qitem query_queue%ROWTYPE; + timeout_ts timestamptz; + error text; + cnt int := 0; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; + IF NOT FOUND THEN + RETURN cnt; + END IF; + cnt := cnt + 1; + BEGIN + qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); + RAISE NOTICE 'RUNNING QUERY: %', qitem.query; + + EXECUTE qitem.query; + EXCEPTION WHEN others THEN + error := format('%s | %s', SQLERRM, SQLSTATE); + RAISE WARNING '%', error; + END; + INSERT INTO query_queue_history (query, added, finished, error) + VALUES (qitem.query, qitem.added, clock_timestamp(), error); + END LOOP; + RETURN cnt; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.t2s(text) + RETURNS text + LANGUAGE sql + IMMUTABLE PARALLEL SAFE STRICT +AS $function$ + SELECT extract(epoch FROM $1::interval)::text || ' s'; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.update_partition_stats(_partition text, istrigger boolean DEFAULT false) + RETURNS void + LANGUAGE plpgsql + STRICT +AS $function$ +DECLARE + dtrange tstzrange; + edtrange tstzrange; + cdtrange tstzrange; + cedtrange tstzrange; + extent geometry; + collection text; +BEGIN + RAISE NOTICE 'Updating stats for %.', _partition; + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]') + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange; + extent := st_estimatedextent('pgstac', _partition, 'geometry'); + INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) + SELECT _partition, dtrange, edtrange, extent, now() + ON CONFLICT (partition) DO + UPDATE SET + dtrange=EXCLUDED.dtrange, + edtrange=EXCLUDED.edtrange, + spatial=EXCLUDED.spatial, + last_updated=EXCLUDED.last_updated + ; + SELECT + constraint_dtrange, constraint_edtrange, partitions.collection + INTO cdtrange, cedtrange, collection + FROM partitions WHERE partition = _partition; + + RAISE NOTICE 'Checking if we need to modify constraints.'; + IF + (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) + AND NOT istrigger + THEN + RAISE NOTICE 'Modifying Constraints'; + RAISE NOTICE 'Existing % %', cdtrange, cedtrange; + RAISE NOTICE 'New % %', dtrange, edtrange; + PERFORM drop_table_constraints(_partition); + PERFORM create_table_constraints(_partition, dtrange, edtrange); + END IF; + RAISE NOTICE 'Checking if we need to update collection extents.'; + IF get_setting_bool('update_collection_extent') THEN + RAISE NOTICE 'updating collection extent for %', collection; + PERFORM run_or_queue(format($q$ + UPDATE collections + SET content = jsonb_set_lax( + content, + '{extent}'::text[], + collection_extent(%L), + true, + 'use_json_null' + ) WHERE id=%L + ; + $q$, collection, collection)); + ELSE + RAISE NOTICE 'Not updating collection extent for %', collection; + END IF; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.update_partition_stats_q(_partition text, istrigger boolean DEFAULT false) + RETURNS void + LANGUAGE plpgsql +AS $function$ +DECLARE +BEGIN + PERFORM run_or_queue( + format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) + ); +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.all_collections() + RETURNS jsonb + LANGUAGE sql + SET search_path TO 'pgstac', 'public' +AS $function$ + SELECT jsonb_agg(content) FROM collections; +$function$ +; + +CREATE OR REPLACE PROCEDURE pgstac.analyze_items() + LANGUAGE plpgsql +AS $procedure$ +DECLARE + q text; + timeout_ts timestamptz; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + RAISE NOTICE '% % %', clock_timestamp(), timeout_ts, current_setting('statement_timeout', TRUE); + SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q + FROM pg_stat_user_tables + WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; + IF NOT FOUND THEN + EXIT; + END IF; + RAISE NOTICE '%', q; + EXECUTE q; + COMMIT; + RAISE NOTICE '%', queue_timeout(); + END LOOP; +END; +$procedure$ +; + +CREATE OR REPLACE FUNCTION pgstac.check_pgstac_settings(_sysmem text DEFAULT NULL::text) + RETURNS void + LANGUAGE plpgsql + SET search_path TO 'pgstac', 'public' + SET client_min_messages TO 'notice' +AS $function$ +DECLARE + settingval text; + sysmem bigint := pg_size_bytes(_sysmem); + effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); + shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); + work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); + max_connections int := current_setting('max_connections', TRUE); + maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); + seq_page_cost float := current_setting('seq_page_cost', TRUE); + random_page_cost float := current_setting('random_page_cost', TRUE); + temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); + r record; +BEGIN + IF _sysmem IS NULL THEN + RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; + ELSE + IF effective_cache_size < (sysmem * 0.5) THEN + RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); + ELSIF effective_cache_size > (sysmem * 0.75) THEN + RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); + ELSE + RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); + END IF; + + IF shared_buffers < (sysmem * 0.2) THEN + RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); + ELSIF shared_buffers > (sysmem * 0.3) THEN + RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); + ELSE + RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); + END IF; + shared_buffers = sysmem * 0.3; + IF maintenance_work_mem < (sysmem * 0.2) THEN + RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); + ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN + RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); + ELSE + RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); + END IF; + + IF work_mem * max_connections > shared_buffers THEN + RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); + ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN + RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); + ELSE + RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); + END IF; + + IF random_page_cost / seq_page_cost != 1.1 THEN + RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; + ELSE + RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; + END IF; + + IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN + RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); + END IF; + END IF; + + RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; + RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; + + FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP + RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; + END LOOP; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; + ELSE + RAISE NOTICE 'pg_cron % is installed', settingval; + END IF; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; + ELSE + RAISE NOTICE 'pgstattuple % is installed', settingval; + END IF; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; + ELSE + RAISE NOTICE 'pg_stat_statements % is installed', settingval; + IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN + RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; + END IF; + END IF; + +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.collections_trigger_func() + RETURNS trigger + LANGUAGE plpgsql +AS $function$ +DECLARE + q text; + partition_name text := format('_items_%s', NEW.key); + partition_exists boolean := false; + partition_empty boolean := true; + err_context text; + loadtemp boolean := FALSE; +BEGIN + RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN + PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); + END IF; + RETURN NEW; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.get_item(_id text, _collection text DEFAULT NULL::text) + RETURNS jsonb + LANGUAGE sql + STABLE + SET search_path TO 'pgstac', 'public' +AS $function$ + SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.get_setting(_setting text, conf jsonb DEFAULT NULL::jsonb) + RETURNS text + LANGUAGE sql +AS $function$ +SELECT COALESCE( + nullif(conf->>_setting, ''), + nullif(current_setting(concat('pgstac.',_setting), TRUE),''), + nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') +); +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.item_by_id(_id text, _collection text DEFAULT NULL::text) + RETURNS pgstac.items + LANGUAGE plpgsql + STABLE + SET search_path TO 'pgstac', 'public' +AS $function$ +DECLARE + i items%ROWTYPE; +BEGIN + SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; + RETURN i; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.items_staging_triggerfunc() + RETURNS trigger + LANGUAGE plpgsql +AS $function$ +DECLARE + p record; + _partitions text[]; + part text; + ts timestamptz := clock_timestamp(); +BEGIN + RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; + + FOR part IN WITH t AS ( + SELECT + n.content->>'collection' as collection, + stac_daterange(n.content->'properties') as dtr, + partition_trunc + FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) + ), p AS ( + SELECT + collection, + COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, + tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, + tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange + FROM t + GROUP BY 1,2 + ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP + RAISE NOTICE 'Partition %', part; + END LOOP; + + RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; + IF TG_TABLE_NAME = 'items_staging' THEN + INSERT INTO items + SELECT + (content_dehydrate(content)).* + FROM newdata; + RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; + DELETE FROM items_staging; + ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN + INSERT INTO items + SELECT + (content_dehydrate(content)).* + FROM newdata + ON CONFLICT DO NOTHING; + RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; + DELETE FROM items_staging_ignore; + ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN + WITH staging_formatted AS ( + SELECT (content_dehydrate(content)).* FROM newdata + ), deletes AS ( + DELETE FROM items i USING staging_formatted s + WHERE + i.id = s.id + AND i.collection = s.collection + AND i IS DISTINCT FROM s + RETURNING i.id, i.collection + ) + INSERT INTO items + SELECT s.* FROM + staging_formatted s + ON CONFLICT DO NOTHING; + RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; + DELETE FROM items_staging_upsert; + END IF; + RAISE NOTICE 'Done. %', clock_timestamp() - ts; + + RETURN NULL; + +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.parse_dtrange(_indate jsonb, relative_base timestamp with time zone DEFAULT date_trunc('hour'::text, CURRENT_TIMESTAMP)) + RETURNS tstzrange + LANGUAGE plpgsql + STABLE PARALLEL SAFE STRICT + SET "TimeZone" TO 'UTC' +AS $function$ +DECLARE + timestrs text[]; + s timestamptz; + e timestamptz; +BEGIN + timestrs := + CASE + WHEN _indate ? 'timestamp' THEN + ARRAY[_indate->>'timestamp'] + WHEN _indate ? 'interval' THEN + to_text_array(_indate->'interval') + WHEN jsonb_typeof(_indate) = 'array' THEN + to_text_array(_indate) + ELSE + regexp_split_to_array( + _indate->>0, + '/' + ) + END; + RAISE NOTICE 'TIMESTRS %', timestrs; + IF cardinality(timestrs) = 1 THEN + IF timestrs[1] ILIKE 'P%' THEN + RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); + END IF; + s := timestrs[1]::timestamptz; + RETURN tstzrange(s, s, '[]'); + END IF; + + IF cardinality(timestrs) != 2 THEN + RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; + END IF; + + IF timestrs[1] = '..' OR timestrs[1] = '' THEN + s := '-infinity'::timestamptz; + e := timestrs[2]::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] = '..' OR timestrs[2] = '' THEN + s := timestrs[1]::timestamptz; + e := 'infinity'::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN + e := timestrs[2]::timestamptz; + s := e - upper(timestrs[1])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN + s := timestrs[1]::timestamptz; + e := s + upper(timestrs[2])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + s := timestrs[1]::timestamptz; + e := timestrs[2]::timestamptz; + + RETURN tstzrange(s,e,'[)'); + + RETURN NULL; + +END; +$function$ +; + +create or replace view "pgstac"."partition_steps" as SELECT partitions.partition AS name, + date_trunc('month'::text, lower(partitions.partition_dtrange)) AS sdate, + (date_trunc('month'::text, upper(partitions.partition_dtrange)) + '1 mon'::interval) AS edate + FROM pgstac.partitions + WHERE ((partitions.partition_dtrange IS NOT NULL) AND (partitions.partition_dtrange <> 'empty'::tstzrange)) + ORDER BY partitions.dtrange; + + +CREATE OR REPLACE FUNCTION pgstac.queryables_trigger_func() + RETURNS trigger + LANGUAGE plpgsql +AS $function$ +DECLARE +BEGIN + PERFORM maintain_partitions(); + RETURN NULL; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) + RETURNS jsonb + LANGUAGE plpgsql + SET search_path TO 'pgstac', 'public' + SET cursor_tuple_fraction TO '1' +AS $function$ +DECLARE + searches searches%ROWTYPE; + _where text; + token_where text; + full_where text; + orderby text; + query text; + token_type text := substr(_search->>'token',1,4); + _limit int := coalesce((_search->>'limit')::int, 10); + curs refcursor; + cntr int := 0; + iter_record items%ROWTYPE; + first_record jsonb; + first_item items%ROWTYPE; + last_item items%ROWTYPE; + last_record jsonb; + out_records jsonb := '[]'::jsonb; + prev_query text; + next text; + prev_id text; + has_next boolean := false; + has_prev boolean := false; + prev text; + total_count bigint; + context jsonb; + collection jsonb; + includes text[]; + excludes text[]; + exit_flag boolean := FALSE; + batches int := 0; + timer timestamptz := clock_timestamp(); + pstart timestamptz; + pend timestamptz; + pcurs refcursor; + search_where search_wheres%ROWTYPE; + id text; +BEGIN +CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; +-- if ids is set, short circuit and just use direct ids query for each id +-- skip any paging or caching +-- hard codes ordering in the same order as the array of ids +IF _search ? 'ids' THEN + INSERT INTO results (content) + SELECT + CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN + content_nonhydrated(items, _search->'fields') + ELSE + content_hydrate(items, _search->'fields') + END + FROM items WHERE + items.id = ANY(to_text_array(_search->'ids')) + AND + CASE WHEN _search ? 'collections' THEN + items.collection = ANY(to_text_array(_search->'collections')) + ELSE TRUE + END + ORDER BY items.datetime desc, items.id desc + ; + SELECT INTO total_count count(*) FROM results; +ELSE + searches := search_query(_search); + _where := searches._where; + orderby := searches.orderby; + search_where := where_stats(_where); + total_count := coalesce(search_where.total_count, search_where.estimated_count); + + IF token_type='prev' THEN + token_where := get_token_filter(_search, null::jsonb); + orderby := sort_sqlorderby(_search, TRUE); + END IF; + IF token_type='next' THEN + token_where := get_token_filter(_search, null::jsonb); + END IF; + + full_where := concat_ws(' AND ', _where, token_where); + RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; + timer := clock_timestamp(); + + FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP + timer := clock_timestamp(); + query := format('%s LIMIT %s', query, _limit + 1); + RAISE NOTICE 'Partition Query: %', query; + batches := batches + 1; + -- curs = create_cursor(query); + RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); + OPEN curs FOR EXECUTE query; + LOOP + FETCH curs into iter_record; + EXIT WHEN NOT FOUND; + cntr := cntr + 1; + + IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN + last_record := content_nonhydrated(iter_record, _search->'fields'); + ELSE + last_record := content_hydrate(iter_record, _search->'fields'); + END IF; + last_item := iter_record; + IF cntr = 1 THEN + first_item := last_item; + first_record := last_record; + END IF; + IF cntr <= _limit THEN + INSERT INTO results (content) VALUES (last_record); + ELSIF cntr > _limit THEN + has_next := true; + exit_flag := true; + EXIT; + END IF; + END LOOP; + CLOSE curs; + RAISE NOTICE 'Query took %.', clock_timestamp()-timer; + timer := clock_timestamp(); + EXIT WHEN exit_flag; + END LOOP; + RAISE NOTICE 'Scanned through % partitions.', batches; +END IF; + +WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) +SELECT jsonb_agg(content) INTO out_records FROM ordered; + +DROP TABLE results; + + +-- Flip things around if this was the result of a prev token query +IF token_type='prev' THEN + out_records := flip_jsonb_array(out_records); + first_item := last_item; + first_record := last_record; +END IF; + +-- If this query has a token, see if there is data before the first record +IF _search ? 'token' THEN + prev_query := format( + 'SELECT 1 FROM items WHERE %s LIMIT 1', + concat_ws( + ' AND ', + _where, + trim(get_token_filter(_search, to_jsonb(first_item))) + ) + ); + RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; + EXECUTE prev_query INTO has_prev; + IF FOUND and has_prev IS NOT NULL THEN + RAISE NOTICE 'Query results from prev query: %', has_prev; + has_prev := TRUE; + END IF; +END IF; +has_prev := COALESCE(has_prev, FALSE); + +IF has_prev THEN + prev := out_records->0->>'id'; +END IF; +IF has_next OR token_type='prev' THEN + next := out_records->-1->>'id'; +END IF; + +IF context(_search->'conf') != 'off' THEN + context := jsonb_strip_nulls(jsonb_build_object( + 'limit', _limit, + 'matched', total_count, + 'returned', coalesce(jsonb_array_length(out_records), 0) + )); +ELSE + context := jsonb_strip_nulls(jsonb_build_object( + 'limit', _limit, + 'returned', coalesce(jsonb_array_length(out_records), 0) + )); +END IF; + +collection := jsonb_build_object( + 'type', 'FeatureCollection', + 'features', coalesce(out_records, '[]'::jsonb), + 'next', next, + 'prev', prev, + 'context', context +); + +RETURN collection; +END; +$function$ +; + +CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) + RETURNS pgstac.searches + LANGUAGE plpgsql + SECURITY DEFINER +AS $function$ +DECLARE + search searches%ROWTYPE; + pexplain jsonb; + t timestamptz; + i interval; +BEGIN + SELECT * INTO search FROM searches + WHERE hash=search_hash(_search, _metadata) FOR UPDATE; + + -- Calculate the where clause if not already calculated + IF search._where IS NULL THEN + search._where := stac_search_to_where(_search); + END IF; + + -- Calculate the order by clause if not already calculated + IF search.orderby IS NULL THEN + search.orderby := sort_sqlorderby(_search); + END IF; + + PERFORM where_stats(search._where, updatestats, _search->'conf'); + + search.lastused := now(); + search.usecount := coalesce(search.usecount, 0) + 1; + INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) + VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) + ON CONFLICT (hash) DO + UPDATE SET + _where = EXCLUDED._where, + orderby = EXCLUDED.orderby, + lastused = EXCLUDED.lastused, + usecount = EXCLUDED.usecount, + metadata = EXCLUDED.metadata + RETURNING * INTO search + ; + RETURN search; + +END; +$function$ +; + +CREATE OR REPLACE PROCEDURE pgstac.validate_constraints() + LANGUAGE plpgsql +AS $procedure$ +DECLARE + q text; +BEGIN + FOR q IN + SELECT + FORMAT( + 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', + nsp.nspname, + cls.relname, + con.conname + ) + + FROM pg_constraint AS con + JOIN pg_class AS cls + ON con.conrelid = cls.oid + JOIN pg_namespace AS nsp + ON cls.relnamespace = nsp.oid + WHERE convalidated = FALSE AND contype in ('c','f') + AND nsp.nspname = 'pgstac' + LOOP + RAISE NOTICE '%', q; + PERFORM run_or_queue(q); + COMMIT; + END LOOP; +END; +$procedure$ +; + +CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) + RETURNS pgstac.search_wheres + LANGUAGE plpgsql + SECURITY DEFINER +AS $function$ +DECLARE + t timestamptz; + i interval; + explain_json jsonb; + partitions text[]; + sw search_wheres%ROWTYPE; + inwhere_hash text := md5(inwhere); + _context text := lower(context(conf)); + _stats_ttl interval := context_stats_ttl(conf); + _estimated_cost float := context_estimated_cost(conf); + _estimated_count int := context_estimated_count(conf); +BEGIN + IF _context = 'off' THEN + sw._where := inwhere; + return sw; + END IF; + + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; + + -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired + IF NOT updatestats THEN + RAISE NOTICE 'Checking if update is needed for: % .', inwhere; + RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; + RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; + RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; + IF + sw.statslastupdated IS NULL + OR (now() - sw.statslastupdated) > _stats_ttl + OR (context(conf) != 'off' AND sw.total_count IS NULL) + THEN + updatestats := TRUE; + END IF; + END IF; + + sw._where := inwhere; + sw.lastused := now(); + sw.usecount := coalesce(sw.usecount,0) + 1; + + IF NOT updatestats THEN + UPDATE search_wheres SET + lastused = sw.lastused, + usecount = sw.usecount + WHERE md5(_where) = inwhere_hash + RETURNING * INTO sw + ; + RETURN sw; + END IF; + + -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query + t := clock_timestamp(); + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) + INTO explain_json; + RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; + i := clock_timestamp() - t; + + sw.statslastupdated := now(); + sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; + sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; + sw.time_to_estimate := extract(epoch from i); + + RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; + RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; + + -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough + IF + _context = 'on' + OR + ( _context = 'auto' AND + ( + sw.estimated_count < _estimated_count + AND + sw.estimated_cost < _estimated_cost + ) + ) + THEN + t := clock_timestamp(); + RAISE NOTICE 'Calculating actual count...'; + EXECUTE format( + 'SELECT count(*) FROM items WHERE %s', + inwhere + ) INTO sw.total_count; + i := clock_timestamp() - t; + RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; + sw.time_to_count := extract(epoch FROM i); + ELSE + sw.total_count := NULL; + sw.time_to_count := NULL; + END IF; + + + INSERT INTO search_wheres + (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) + SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count + ON CONFLICT ((md5(_where))) + DO UPDATE + SET + lastused = sw.lastused, + usecount = sw.usecount, + statslastupdated = sw.statslastupdated, + estimated_count = sw.estimated_count, + estimated_cost = sw.estimated_cost, + time_to_estimate = sw.time_to_estimate, + total_count = sw.total_count, + time_to_count = sw.time_to_count + ; + RETURN sw; +END; +$function$ +; + +CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON pgstac.items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION pgstac.partition_after_triggerfunc(); + +CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON pgstac.items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION pgstac.partition_after_triggerfunc(); + +CREATE TRIGGER items_after_update_trigger AFTER DELETE ON pgstac.items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION pgstac.partition_after_triggerfunc(); + + + +-- END migra calculated SQL +INSERT INTO queryables (name, definition) VALUES +('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), +('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), +('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') +ON CONFLICT DO NOTHING; + +INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES +('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') +ON CONFLICT DO NOTHING; + + +INSERT INTO pgstac_settings (name, value) VALUES + ('context', 'off'), + ('context_estimated_count', '100000'), + ('context_estimated_cost', '100000'), + ('context_stats_ttl', '1 day'), + ('default_filter_lang', 'cql2-json'), + ('additional_properties', 'true'), + ('use_queue', 'false'), + ('queue_timeout', '10 minutes'), + ('update_collection_extent', 'true') +ON CONFLICT DO NOTHING +; + + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +GRANT ALL ON SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON SCHEMA pgstac to pgstac_admin; + +-- pgstac_read role limited to using function apis +GRANT EXECUTE ON FUNCTION search TO pgstac_read; +GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; +GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; +GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; + +SELECT update_partition_stats_q(partition) FROM partitions; +SELECT set_version('0.7.0'); diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.13.sql b/src/pgstac/migrations/pgstac.0.6.13.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.13.sql rename to src/pgstac/migrations/pgstac.0.6.13.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.2-0.6.3.sql b/src/pgstac/migrations/pgstac.0.6.2-0.6.3.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.2-0.6.3.sql rename to src/pgstac/migrations/pgstac.0.6.2-0.6.3.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.2.sql b/src/pgstac/migrations/pgstac.0.6.2.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.2.sql rename to src/pgstac/migrations/pgstac.0.6.2.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.3-0.6.4.sql b/src/pgstac/migrations/pgstac.0.6.3-0.6.4.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.3-0.6.4.sql rename to src/pgstac/migrations/pgstac.0.6.3-0.6.4.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.3.sql b/src/pgstac/migrations/pgstac.0.6.3.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.3.sql rename to src/pgstac/migrations/pgstac.0.6.3.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.4-0.6.5.sql b/src/pgstac/migrations/pgstac.0.6.4-0.6.5.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.4-0.6.5.sql rename to src/pgstac/migrations/pgstac.0.6.4-0.6.5.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.4.sql b/src/pgstac/migrations/pgstac.0.6.4.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.4.sql rename to src/pgstac/migrations/pgstac.0.6.4.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.5-0.6.6.sql b/src/pgstac/migrations/pgstac.0.6.5-0.6.6.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.5-0.6.6.sql rename to src/pgstac/migrations/pgstac.0.6.5-0.6.6.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.5.sql b/src/pgstac/migrations/pgstac.0.6.5.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.5.sql rename to src/pgstac/migrations/pgstac.0.6.5.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.6-0.6.7.sql b/src/pgstac/migrations/pgstac.0.6.6-0.6.7.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.6-0.6.7.sql rename to src/pgstac/migrations/pgstac.0.6.6-0.6.7.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.6.sql b/src/pgstac/migrations/pgstac.0.6.6.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.6.sql rename to src/pgstac/migrations/pgstac.0.6.6.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.7-0.6.8.sql b/src/pgstac/migrations/pgstac.0.6.7-0.6.8.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.7-0.6.8.sql rename to src/pgstac/migrations/pgstac.0.6.7-0.6.8.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.7.sql b/src/pgstac/migrations/pgstac.0.6.7.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.7.sql rename to src/pgstac/migrations/pgstac.0.6.7.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.8-0.6.9.sql b/src/pgstac/migrations/pgstac.0.6.8-0.6.9.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.8-0.6.9.sql rename to src/pgstac/migrations/pgstac.0.6.8-0.6.9.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.8.sql b/src/pgstac/migrations/pgstac.0.6.8.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.8.sql rename to src/pgstac/migrations/pgstac.0.6.8.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.9-0.6.10.sql b/src/pgstac/migrations/pgstac.0.6.9-0.6.10.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.9-0.6.10.sql rename to src/pgstac/migrations/pgstac.0.6.9-0.6.10.sql diff --git a/pypgstac/pypgstac/migrations/pgstac.0.6.9.sql b/src/pgstac/migrations/pgstac.0.6.9.sql similarity index 100% rename from pypgstac/pypgstac/migrations/pgstac.0.6.9.sql rename to src/pgstac/migrations/pgstac.0.6.9.sql diff --git a/src/pgstac/migrations/pgstac.0.7.0.sql b/src/pgstac/migrations/pgstac.0.7.0.sql new file mode 100644 index 00000000..d7f7fe5b --- /dev/null +++ b/src/pgstac/migrations/pgstac.0.7.0.sql @@ -0,0 +1,3707 @@ +DO $$ +DECLARE +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN + CREATE EXTENSION IF NOT EXISTS postgis; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN + CREATE EXTENSION IF NOT EXISTS btree_gist; + END IF; +END; +$$ LANGUAGE PLPGSQL; + +DO $$ + BEGIN + CREATE ROLE pgstac_admin; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_read; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_ingest; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + + +GRANT pgstac_admin TO current_user; + +-- Function to make sure pgstac_admin is the owner of items +CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ +DECLARE + f RECORD; +BEGIN + FOR f IN ( + SELECT + concat( + oid::regproc::text, + '(', + coalesce(pg_get_function_identity_arguments(oid),''), + ')' + ) AS name, + CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ + FROM pg_proc + WHERE + pronamespace=to_regnamespace('pgstac') + AND proowner != to_regrole('pgstac_admin') + AND proname NOT LIKE 'pg_stat%' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + FOR f IN ( + SELECT + oid::regclass::text as name, + CASE relkind + WHEN 'i' THEN 'INDEX' + WHEN 'I' THEN 'INDEX' + WHEN 'p' THEN 'TABLE' + WHEN 'r' THEN 'TABLE' + WHEN 'v' THEN 'VIEW' + WHEN 'S' THEN 'SEQUENCE' + ELSE NULL + END as typ + FROM pg_class + WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + RETURN; +END; +$$ LANGUAGE PLPGSQL; +SELECT pgstac_admin_owns(); + +CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; + +GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; + +ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; + +GRANT pgstac_read TO pgstac_ingest; +GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; + +SET ROLE pgstac_admin; + +SET SEARCH_PATH TO pgstac, public; + +DROP FUNCTION IF EXISTS analyze_items; +DROP FUNCTION IF EXISTS validate_constraints; +CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ + SELECT floor(($1->>0)::float)::int; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ + SELECT ($1->>0)::float; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ + SELECT ($1->>0)::timestamptz; +$$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; + + +CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ + SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ + SELECT + CASE jsonb_typeof($1) + WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) + ELSE ARRAY[$1->>0] + END + ; +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ +SELECT CASE jsonb_array_length(_bbox) + WHEN 4 THEN + ST_SetSRID(ST_MakeEnvelope( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float, + (_bbox->>3)::float + ),4326) + WHEN 6 THEN + ST_SetSRID(ST_3DMakeBox( + ST_MakePoint( + (_bbox->>0)::float, + (_bbox->>1)::float, + (_bbox->>2)::float + ), + ST_MakePoint( + (_bbox->>3)::float, + (_bbox->>4)::float, + (_bbox->>5)::float + ) + ),4326) + ELSE null END; +; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ + SELECT jsonb_build_array( + st_xmin(_geom), + st_ymin(_geom), + st_xmax(_geom), + st_ymax(_geom) + ); +$$ LANGUAGE SQL IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ + SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ + SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ + WITH RECURSIVE t AS ( + SELECT e FROM explode_dotpaths(j) e + UNION ALL + SELECT e[1:cardinality(e)-1] + FROM t + WHERE cardinality(e)>1 + ) SELECT e FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ +DECLARE +BEGIN + IF cardinality(path) > 1 THEN + FOR i IN 1..(cardinality(path)-1) LOOP + IF j #> path[:i] IS NULL THEN + j := jsonb_set_lax(j, path[:i], '{}', TRUE); + END IF; + END LOOP; + END IF; + RETURN jsonb_set_lax(j, path, val, true); + +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + + + +CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ +DECLARE + includes jsonb := f-> 'include'; + outj jsonb := '{}'::jsonb; + path text[]; +BEGIN + IF + includes IS NULL + OR jsonb_array_length(includes) = 0 + THEN + RETURN j; + ELSE + includes := includes || '["id","collection"]'::jsonb; + FOR path IN SELECT explode_dotpaths(includes) LOOP + outj := jsonb_set_nested(outj, path, j #> path); + END LOOP; + END IF; + RETURN outj; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ +DECLARE + excludes jsonb := f-> 'exclude'; + outj jsonb := j; + path text[]; +BEGIN + IF + excludes IS NULL + OR jsonb_array_length(excludes) = 0 + THEN + RETURN j; + ELSE + FOR path IN SELECT explode_dotpaths(excludes) LOOP + outj := outj #- path; + END LOOP; + END IF; + RETURN outj; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ + SELECT jsonb_exclude(jsonb_include(j, f), f); +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ + SELECT + CASE + WHEN _a = '"𒍟※"'::jsonb THEN NULL + WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b + WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN + ( + SELECT + jsonb_strip_nulls( + jsonb_object_agg( + key, + merge_jsonb(a.value, b.value) + ) + ) + FROM + jsonb_each(coalesce(_a,'{}'::jsonb)) as a + FULL JOIN + jsonb_each(coalesce(_b,'{}'::jsonb)) as b + USING (key) + ) + WHEN + jsonb_typeof(_a) = 'array' + AND jsonb_typeof(_b) = 'array' + AND jsonb_array_length(_a) = jsonb_array_length(_b) + THEN + ( + SELECT jsonb_agg(m) FROM + ( SELECT + merge_jsonb( + jsonb_array_elements(_a), + jsonb_array_elements(_b) + ) as m + ) as l + ) + ELSE _a + END + ; +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ + SELECT + CASE + + WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb + WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a + WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb + WHEN _a = _b THEN NULL + WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN + ( + SELECT + jsonb_strip_nulls( + jsonb_object_agg( + key, + strip_jsonb(a.value, b.value) + ) + ) + FROM + jsonb_each(_a) as a + FULL JOIN + jsonb_each(_b) as b + USING (key) + ) + WHEN + jsonb_typeof(_a) = 'array' + AND jsonb_typeof(_b) = 'array' + AND jsonb_array_length(_a) = jsonb_array_length(_b) + THEN + ( + SELECT jsonb_agg(m) FROM + ( SELECT + strip_jsonb( + jsonb_array_elements(_a), + jsonb_array_elements(_b) + ) as m + ) as l + ) + ELSE _a + END + ; +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ + SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ + SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ + SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ + SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ + SELECT nullif_jsonbnullempty(greatest(a, b)); +$$ LANGUAGE SQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ + SELECT COALESCE($1,$2); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( + SFUNC = first_notnull_sfunc, + STYPE = anyelement +); + +CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( + STYPE = jsonb, + SFUNC = jsonb_concat_ignorenull, + FINALFUNC = jsonb_array_unique +); + +CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( + STYPE = jsonb, + SFUNC = jsonb_least +); + +CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( + STYPE = jsonb, + SFUNC = jsonb_greatest +); + +CREATE TABLE IF NOT EXISTS migrations ( + version text PRIMARY KEY, + datetime timestamptz DEFAULT clock_timestamp() NOT NULL +); + +CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ + SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ + INSERT INTO pgstac.migrations (version) VALUES ($1) + ON CONFLICT DO NOTHING + RETURNING version; +$$ LANGUAGE SQL; + + +CREATE TABLE IF NOT EXISTS pgstac_settings ( + name text PRIMARY KEY, + value text NOT NULL +); + +CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ +DECLARE + retval boolean; +BEGIN + EXECUTE format($q$ + SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) + $q$, + $1 + ) INTO retval; + RETURN retval; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ +SELECT COALESCE( + nullif(conf->>_setting, ''), + nullif(current_setting(concat('pgstac.',_setting), TRUE),''), + nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') +); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ +SELECT COALESCE( + conf->>_setting, + current_setting(concat('pgstac.',_setting), TRUE), + (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), + 'FALSE' +)::boolean; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ + SELECT pgstac.get_setting('context', conf); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ + SELECT pgstac.get_setting('context_estimated_count', conf)::int; +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS context_estimated_cost(); +CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ + SELECT pgstac.get_setting('context_estimated_cost', conf)::float; +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS context_stats_ttl(); +CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ + SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ + SELECT extract(epoch FROM $1::interval)::text || ' s'; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; + +CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ + SELECT set_config( + 'statement_timeout', + t2s(coalesce( + get_setting('queue_timeout'), + '1h' + )), + false + )::interval; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ +DECLARE +debug boolean := current_setting('pgstac.debug', true); +BEGIN + IF debug THEN + RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); + RETURN TRUE; + END IF; + RETURN FALSE; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ +SELECT CASE + WHEN $1 IS NULL THEN TRUE + WHEN cardinality($1)<1 THEN TRUE +ELSE FALSE +END; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ + SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); +$$ LANGUAGE SQL IMMUTABLE; + + +CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) + RETURNS text[] AS $$ + SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) + RETURNS text[] AS $$ + SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; +$$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ +SELECT ARRAY( + SELECT $1[i] + FROM generate_subscripts($1,1) AS s(i) + ORDER BY i DESC +); +$$ LANGUAGE SQL STRICT IMMUTABLE; + +DROP TABLE IF EXISTS query_queue; +CREATE TABLE query_queue ( + query text PRIMARY KEY, + added timestamptz DEFAULT now() +); + +DROP TABLE IF EXISTS query_queue_history; +CREATE TABLE query_queue_history( + query text, + added timestamptz NOT NULL, + finished timestamptz NOT NULL DEFAULT now(), + error text +); + +CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ +DECLARE + qitem query_queue%ROWTYPE; + timeout_ts timestamptz; + error text; + cnt int := 0; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; + IF NOT FOUND THEN + EXIT; + END IF; + cnt := cnt + 1; + BEGIN + RAISE NOTICE 'RUNNING QUERY: %', qitem.query; + EXECUTE qitem.query; + EXCEPTION WHEN others THEN + error := format('%s | %s', SQLERRM, SQLSTATE); + END; + INSERT INTO query_queue_history (query, added, finished, error) + VALUES (qitem.query, qitem.added, clock_timestamp(), error); + COMMIT; + END LOOP; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ +DECLARE + qitem query_queue%ROWTYPE; + timeout_ts timestamptz; + error text; + cnt int := 0; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; + IF NOT FOUND THEN + RETURN cnt; + END IF; + cnt := cnt + 1; + BEGIN + qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); + RAISE NOTICE 'RUNNING QUERY: %', qitem.query; + + EXECUTE qitem.query; + EXCEPTION WHEN others THEN + error := format('%s | %s', SQLERRM, SQLSTATE); + RAISE WARNING '%', error; + END; + INSERT INTO query_queue_history (query, added, finished, error) + VALUES (qitem.query, qitem.added, clock_timestamp(), error); + END LOOP; + RETURN cnt; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ +DECLARE + use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; +BEGIN + IF get_setting_bool('debug') THEN + RAISE NOTICE '%', query; + END IF; + IF use_queue THEN + INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; + ELSE + EXECUTE query; + END IF; + RETURN; +END; +$$ LANGUAGE PLPGSQL; + + + +DROP FUNCTION IF EXISTS check_pgstac_settings; +CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ +DECLARE + settingval text; + sysmem bigint := pg_size_bytes(_sysmem); + effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); + shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); + work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); + max_connections int := current_setting('max_connections', TRUE); + maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); + seq_page_cost float := current_setting('seq_page_cost', TRUE); + random_page_cost float := current_setting('random_page_cost', TRUE); + temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); + r record; +BEGIN + IF _sysmem IS NULL THEN + RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; + ELSE + IF effective_cache_size < (sysmem * 0.5) THEN + RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); + ELSIF effective_cache_size > (sysmem * 0.75) THEN + RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); + ELSE + RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); + END IF; + + IF shared_buffers < (sysmem * 0.2) THEN + RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); + ELSIF shared_buffers > (sysmem * 0.3) THEN + RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); + ELSE + RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); + END IF; + shared_buffers = sysmem * 0.3; + IF maintenance_work_mem < (sysmem * 0.2) THEN + RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); + ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN + RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); + ELSE + RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); + END IF; + + IF work_mem * max_connections > shared_buffers THEN + RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); + ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN + RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); + ELSE + RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); + END IF; + + IF random_page_cost / seq_page_cost != 1.1 THEN + RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; + ELSE + RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; + END IF; + + IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN + RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); + END IF; + END IF; + + RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; + RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; + + FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP + RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; + END LOOP; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; + ELSE + RAISE NOTICE 'pg_cron % is installed', settingval; + END IF; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; + ELSE + RAISE NOTICE 'pgstattuple % is installed', settingval; + END IF; + + SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; + IF NOT FOUND OR settingval IS NULL THEN + RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; + ELSE + RAISE NOTICE 'pg_stat_statements % is installed', settingval; + IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN + RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; + END IF; + END IF; + +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; +/* looks for a geometry in a stac item first from geometry and falling back to bbox */ +CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ +SELECT + CASE + WHEN value ? 'intersects' THEN + ST_GeomFromGeoJSON(value->>'intersects') + WHEN value ? 'geometry' THEN + ST_GeomFromGeoJSON(value->>'geometry') + WHEN value ? 'bbox' THEN + pgstac.bbox_geom(value->'bbox') + ELSE NULL + END as geometry +; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION stac_daterange( + value jsonb +) RETURNS tstzrange AS $$ +DECLARE + props jsonb := value; + dt timestamptz; + edt timestamptz; +BEGIN + IF props ? 'properties' THEN + props := props->'properties'; + END IF; + IF + props ? 'start_datetime' + AND props->>'start_datetime' IS NOT NULL + AND props ? 'end_datetime' + AND props->>'end_datetime' IS NOT NULL + THEN + dt := props->>'start_datetime'; + edt := props->>'end_datetime'; + IF dt > edt THEN + RAISE EXCEPTION 'start_datetime must be < end_datetime'; + END IF; + ELSE + dt := props->>'datetime'; + edt := props->>'datetime'; + END IF; + IF dt is NULL OR edt IS NULL THEN + RAISE NOTICE 'DT: %, EDT: %', dt, edt; + RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; + END IF; + RETURN tstzrange(dt, edt, '[]'); +END; +$$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ + SELECT lower(stac_daterange(value)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ + SELECT upper(stac_daterange(value)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; + +CREATE TABLE IF NOT EXISTS stac_extensions( + url text PRIMARY KEY, + content jsonb +); +CREATE TABLE queryables ( + id bigint GENERATED ALWAYS AS identity PRIMARY KEY, + name text NOT NULL, + collection_ids text[], -- used to determine what partitions to create indexes on + definition jsonb, + property_path text, + property_wrapper text, + property_index_type text +); +CREATE INDEX queryables_name_idx ON queryables (name); +CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); + + + + +CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ + SELECT string_agg( + quote_literal(v), + '->' + ) FROM unnest(arr) v; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + + + +CREATE OR REPLACE FUNCTION queryable( + IN dotpath text, + OUT path text, + OUT expression text, + OUT wrapper text, + OUT nulled_wrapper text +) AS $$ +DECLARE + q RECORD; + path_elements text[]; +BEGIN + IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN + path := dotpath; + expression := dotpath; + wrapper := NULL; + RETURN; + END IF; + SELECT * INTO q FROM queryables + WHERE + name=dotpath + OR name = 'properties.' || dotpath + OR name = replace(dotpath, 'properties.', '') + ; + IF q.property_wrapper IS NULL THEN + IF q.definition->>'type' = 'number' THEN + wrapper := 'to_float'; + nulled_wrapper := wrapper; + ELSIF q.definition->>'format' = 'date-time' THEN + wrapper := 'to_tstz'; + nulled_wrapper := wrapper; + ELSE + nulled_wrapper := NULL; + wrapper := 'to_text'; + END IF; + ELSE + wrapper := q.property_wrapper; + nulled_wrapper := wrapper; + END IF; + IF q.property_path IS NOT NULL THEN + path := q.property_path; + ELSE + path_elements := string_to_array(dotpath, '.'); + IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN + path := format('content->%s', array_to_path(path_elements)); + ELSIF path_elements[1] = 'properties' THEN + path := format('content->%s', array_to_path(path_elements)); + ELSE + path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); + END IF; + END IF; + expression := format('%I(%s)', wrapper, path); + RETURN; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + + +DROP VIEW IF EXISTS pgstac_index; +CREATE VIEW pgstac_indexes AS +SELECT + i.schemaname, + i.tablename, + i.indexname, + indexdef, + COALESCE( + (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], + (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], + CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END + ) AS field, + pg_table_size(i.indexname::text) as index_size, + pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty +FROM + pg_indexes i +WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; + +DROP VIEW IF EXISTS pgstac_index_stats; +CREATE VIEW pgstac_indexes_stats AS +SELECT + i.schemaname, + i.tablename, + i.indexname, + indexdef, + COALESCE( + (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], + (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], + CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END + ) AS field, + pg_table_size(i.indexname::text) as index_size, + pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, + n_distinct, + most_common_vals::text::text[], + most_common_freqs::text::text[], + histogram_bounds::text::text[], + correlation +FROM + pg_indexes i + LEFT JOIN pg_stats s ON (s.tablename = i.indexname) +WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; + +set check_function_bodies to off; +CREATE OR REPLACE FUNCTION maintain_partition_queries( + part text DEFAULT 'items', + dropindexes boolean DEFAULT FALSE, + rebuildindexes boolean DEFAULT FALSE +) RETURNS SETOF text AS $$ +DECLARE + parent text; + level int; + isleaf bool; + collection collections%ROWTYPE; + subpart text; + baseidx text; + queryable_name text; + queryable_property_index_type text; + queryable_property_wrapper text; + queryable_parsed RECORD; + deletedidx pg_indexes%ROWTYPE; + q text; + idx text; + collection_partition bigint; + _concurrently text := ''; +BEGIN + RAISE NOTICE 'Maintaining partition: %', part; + IF get_setting_bool('use_queue') THEN + _concurrently='CONCURRENTLY'; + END IF; + + -- Get root partition + SELECT parentrelid::text, pt.isleaf, pt.level + INTO parent, isleaf, level + FROM pg_partition_tree('items') pt + WHERE relid::text = part; + IF NOT FOUND THEN + RAISE NOTICE 'Partition % Does Not Exist In Partition Tree', part; + RETURN; + END IF; + + -- If this is a parent partition, recurse to leaves + IF NOT isleaf THEN + FOR subpart IN + SELECT relid::text + FROM pg_partition_tree(part) + WHERE relid::text != part + LOOP + RAISE NOTICE 'Recursing to %', subpart; + RETURN QUERY SELECT * FROM maintain_partition_queries(subpart, dropindexes, rebuildindexes); + END LOOP; + RETURN; -- Don't continue since not an end leaf + END IF; + + + -- Get collection + collection_partition := ((regexp_match(part, E'^_items_([0-9]+)'))[1])::bigint; + RAISE NOTICE 'COLLECTION PARTITION: %', collection_partition; + SELECT * INTO STRICT collection + FROM collections + WHERE key = collection_partition; + RAISE NOTICE 'COLLECTION ID: %s', collection.id; + + + -- Create temp table with existing indexes + CREATE TEMP TABLE existing_indexes ON COMMIT DROP AS + SELECT * + FROM pg_indexes + WHERE schemaname='pgstac' AND tablename=part; + + + -- Check if index exists for each queryable. + FOR + queryable_name, + queryable_property_index_type, + queryable_property_wrapper + IN + SELECT + name, + COALESCE(property_index_type, 'BTREE'), + COALESCE(property_wrapper, 'to_text') + FROM queryables + WHERE + name NOT in ('id', 'datetime', 'geometry') + AND ( + collection_ids IS NULL + OR collection_ids = '{}'::text[] + OR collection.id = ANY (collection_ids) + ) + UNION ALL + SELECT 'datetime desc, end_datetime', 'BTREE', '' + UNION ALL + SELECT 'geometry', 'GIST', '' + UNION ALL + SELECT 'id', 'BTREE', '' + LOOP + baseidx := format( + $q$ ON %I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, + part, + queryable_property_index_type, + queryable_property_wrapper, + queryable_name + ); + RAISE NOTICE 'BASEIDX: %', baseidx; + RAISE NOTICE 'IDXSEARCH: %', format($q$[(']%s[')]$q$, queryable_name); + -- If index already exists, delete it from existing indexes type table + FOR deletedidx IN + DELETE FROM existing_indexes + WHERE indexdef ~* format($q$[(']%s[')]$q$, queryable_name) + RETURNING * + LOOP + RAISE NOTICE 'EXISTING INDEX: %', deletedidx; + IF NOT FOUND THEN -- index did not exist, create it + RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); + ELSIF rebuildindexes THEN + RETURN NEXT format('REINDEX %I %s;', deletedidx.indexname, _concurrently); + END IF; + END LOOP; + END LOOP; + + -- Remove indexes that were not expected + FOR idx IN SELECT indexname::text FROM existing_indexes + LOOP + RAISE WARNING 'Index: % is not defined by queryables.', idx; + IF dropindexes THEN + RETURN NEXT format('DROP INDEX IF EXISTS %I;', idx); + END IF; + END LOOP; + + DROP TABLE existing_indexes; + RAISE NOTICE 'Returning from maintain_partition_queries.'; + RETURN; + +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION maintain_partitions( + part text DEFAULT 'items', + dropindexes boolean DEFAULT FALSE, + rebuildindexes boolean DEFAULT FALSE +) RETURNS VOID AS $$ + WITH t AS ( + SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q + ) SELECT count(*) FROM t; +$$ LANGUAGE SQL; + + +CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ +DECLARE +BEGIN + PERFORM maintain_partitions(); + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL; + +CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables +FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); + + +CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ +DECLARE +BEGIN + -- Build up queryables if the input contains valid collection ids or is empty + IF EXISTS ( + SELECT 1 FROM collections + WHERE + _collection_ids IS NULL + OR cardinality(_collection_ids) = 0 + OR id = ANY(_collection_ids) + ) + THEN + RETURN ( + WITH base AS ( + SELECT + unnest(collection_ids) as collection_id, + name, + coalesce(definition, '{"type":"string"}'::jsonb) as definition + FROM queryables + WHERE + _collection_ids IS NULL OR + _collection_ids = '{}'::text[] OR + _collection_ids && collection_ids + UNION ALL + SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition + FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] + ), g AS ( + SELECT + name, + first_notnull(definition) as definition, + jsonb_array_unique_merge(definition->'enum') as enum, + jsonb_min(definition->'minimum') as minimum, + jsonb_min(definition->'maxiumn') as maximum + FROM base + GROUP BY 1 + ) + SELECT + jsonb_build_object( + '$schema', 'http://json-schema.org/draft-07/schema#', + '$id', '', + 'type', 'object', + 'title', 'STAC Queryables.', + 'properties', jsonb_object_agg( + name, + definition + || + jsonb_strip_nulls(jsonb_build_object( + 'enum', enum, + 'minimum', minimum, + 'maximum', maximum + )) + ) + ) + FROM g + ); + ELSE + RETURN NULL; + END IF; +END; +$$ LANGUAGE PLPGSQL STABLE; + +CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ + SELECT + CASE + WHEN _collection IS NULL THEN get_queryables(NULL::text[]) + ELSE get_queryables(ARRAY[_collection]) + END + ; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ + SELECT get_queryables(NULL::text[]); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ + SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; +$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; + + +CREATE OR REPLACE VIEW stac_extension_queryables AS +SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; + + +CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ +DECLARE + q text; + _partition text; + explain_json json; + psize float; + estrows float; +BEGIN + SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; + + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) + INTO explain_json; + psize := explain_json->0->'Plan'->'Plan Rows'; + estrows := _tablesample * .01 * psize; + IF estrows < minrows THEN + _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); + RAISE NOTICE '%', (psize / estrows) / 100; + END IF; + RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; + + q := format( + $q$ + WITH q AS ( + SELECT * FROM queryables + WHERE + collection_ids IS NULL + OR %L = ANY(collection_ids) + ), t AS ( + SELECT + content->'properties' AS properties + FROM + %I + TABLESAMPLE SYSTEM(%L) + ), p AS ( + SELECT DISTINCT ON (key) + key, + value, + s.definition + FROM t + JOIN LATERAL jsonb_each(properties) ON TRUE + LEFT JOIN q ON (q.name=key) + LEFT JOIN stac_extension_queryables s ON (s.name=key) + WHERE q.definition IS NULL + ) + SELECT + %L, + key, + COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, + CASE + WHEN definition->>'type' = 'integer' THEN 'to_int' + WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' + WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' + ELSE 'to_text' + END + FROM p; + $q$, + _collection, + _partition, + _tablesample, + _collection + ); + RETURN QUERY EXECUTE q; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ + SELECT + array_agg(collection), + name, + definition, + property_wrapper + FROM + collections + JOIN LATERAL + missing_queryables(id, _tablesample) c + ON TRUE + GROUP BY + 2,3,4 + ORDER BY 2,1 + ; +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate jsonb, + relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) +) RETURNS tstzrange AS $$ +DECLARE + timestrs text[]; + s timestamptz; + e timestamptz; +BEGIN + timestrs := + CASE + WHEN _indate ? 'timestamp' THEN + ARRAY[_indate->>'timestamp'] + WHEN _indate ? 'interval' THEN + to_text_array(_indate->'interval') + WHEN jsonb_typeof(_indate) = 'array' THEN + to_text_array(_indate) + ELSE + regexp_split_to_array( + _indate->>0, + '/' + ) + END; + RAISE NOTICE 'TIMESTRS %', timestrs; + IF cardinality(timestrs) = 1 THEN + IF timestrs[1] ILIKE 'P%' THEN + RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); + END IF; + s := timestrs[1]::timestamptz; + RETURN tstzrange(s, s, '[]'); + END IF; + + IF cardinality(timestrs) != 2 THEN + RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; + END IF; + + IF timestrs[1] = '..' OR timestrs[1] = '' THEN + s := '-infinity'::timestamptz; + e := timestrs[2]::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] = '..' OR timestrs[2] = '' THEN + s := timestrs[1]::timestamptz; + e := 'infinity'::timestamptz; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN + e := timestrs[2]::timestamptz; + s := e - upper(timestrs[1])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN + s := timestrs[1]::timestamptz; + e := s + upper(timestrs[2])::interval; + RETURN tstzrange(s,e,'[)'); + END IF; + + s := timestrs[1]::timestamptz; + e := timestrs[2]::timestamptz; + + RETURN tstzrange(s,e,'[)'); + + RETURN NULL; + +END; +$$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION parse_dtrange( + _indate text, + relative_base timestamptz DEFAULT CURRENT_TIMESTAMP +) RETURNS tstzrange AS $$ + SELECT parse_dtrange(to_jsonb(_indate), relative_base); +$$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + ll text := 'datetime'; + lh text := 'end_datetime'; + rrange tstzrange; + rl text; + rh text; + outq text; +BEGIN + rrange := parse_dtrange(args->1); + RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; + op := lower(op); + rl := format('%L::timestamptz', lower(rrange)); + rh := format('%L::timestamptz', upper(rrange)); + outq := CASE op + WHEN 't_before' THEN 'lh < rl' + WHEN 't_after' THEN 'll > rh' + WHEN 't_meets' THEN 'lh = rl' + WHEN 't_metby' THEN 'll = rh' + WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' + WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' + WHEN 't_starts' THEN 'll = rl AND lh < rh' + WHEN 't_startedby' THEN 'll = rl AND lh > rh' + WHEN 't_during' THEN 'll > rl AND lh < rh' + WHEN 't_contains' THEN 'll < rl AND lh > rh' + WHEN 't_finishes' THEN 'll > rl AND lh = rh' + WHEN 't_finishedby' THEN 'll < rl AND lh = rh' + WHEN 't_equals' THEN 'll = rl AND lh = rh' + WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' + WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' + WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' + END; + outq := regexp_replace(outq, '\mll\M', ll); + outq := regexp_replace(outq, '\mlh\M', lh); + outq := regexp_replace(outq, '\mrl\M', rl); + outq := regexp_replace(outq, '\mrh\M', rh); + outq := format('(%s)', outq); + RETURN outq; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + + + +CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ +DECLARE + geom text; + j jsonb := args->1; +BEGIN + op := lower(op); + RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; + IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN + RAISE EXCEPTION 'Spatial Operator % Not Supported', op; + END IF; + op := regexp_replace(op, '^s_', 'st_'); + IF op = 'intersects' THEN + op := 'st_intersects'; + END IF; + -- Convert geometry to WKB string + IF j ? 'type' AND j ? 'coordinates' THEN + geom := st_geomfromgeojson(j)::text; + ELSIF jsonb_typeof(j) = 'array' THEN + geom := bbox_geom(j)::text; + END IF; + + RETURN format('%s(geometry, %L::geometry)', op, geom); +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ +-- Translates anything passed in through the deprecated "query" into equivalent CQL2 +WITH t AS ( + SELECT key as property, value as ops + FROM jsonb_each(q) +), t2 AS ( + SELECT property, (jsonb_each(ops)).* + FROM t WHERE jsonb_typeof(ops) = 'object' + UNION ALL + SELECT property, 'eq', ops + FROM t WHERE jsonb_typeof(ops) != 'object' +) +SELECT + jsonb_strip_nulls(jsonb_build_object( + 'op', 'and', + 'args', jsonb_agg( + jsonb_build_object( + 'op', key, + 'args', jsonb_build_array( + jsonb_build_object('property',property), + value + ) + ) + ) + ) +) as qcql FROM t2 +; +$$ LANGUAGE SQL IMMUTABLE STRICT; + + +CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ +DECLARE + args jsonb; + ret jsonb; +BEGIN + RAISE NOTICE 'CQL1_TO_CQL2: %', j; + IF j ? 'filter' THEN + RETURN cql1_to_cql2(j->'filter'); + END IF; + IF j ? 'property' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'array' THEN + SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; + RETURN args; + END IF; + IF jsonb_typeof(j) = 'number' THEN + RETURN j; + END IF; + IF jsonb_typeof(j) = 'string' THEN + RETURN j; + END IF; + + IF jsonb_typeof(j) = 'object' THEN + SELECT jsonb_build_object( + 'op', key, + 'args', cql1_to_cql2(value) + ) INTO ret + FROM jsonb_each(j) + WHERE j IS NOT NULL; + RETURN ret; + END IF; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE STRICT; + +CREATE TABLE cql2_ops ( + op text PRIMARY KEY, + template text, + types text[] +); +INSERT INTO cql2_ops (op, template, types) VALUES + ('eq', '%s = %s', NULL), + ('neq', '%s != %s', NULL), + ('ne', '%s != %s', NULL), + ('!=', '%s != %s', NULL), + ('<>', '%s != %s', NULL), + ('lt', '%s < %s', NULL), + ('lte', '%s <= %s', NULL), + ('gt', '%s > %s', NULL), + ('gte', '%s >= %s', NULL), + ('le', '%s <= %s', NULL), + ('ge', '%s >= %s', NULL), + ('=', '%s = %s', NULL), + ('<', '%s < %s', NULL), + ('<=', '%s <= %s', NULL), + ('>', '%s > %s', NULL), + ('>=', '%s >= %s', NULL), + ('like', '%s LIKE %s', NULL), + ('ilike', '%s ILIKE %s', NULL), + ('+', '%s + %s', NULL), + ('-', '%s - %s', NULL), + ('*', '%s * %s', NULL), + ('/', '%s / %s', NULL), + ('not', 'NOT (%s)', NULL), + ('between', '%s BETWEEN %s AND %s', NULL), + ('isnull', '%s IS NULL', NULL), + ('upper', 'upper(%s)', NULL), + ('lower', 'lower(%s)', NULL) +ON CONFLICT (op) DO UPDATE + SET + template = EXCLUDED.template +; + + +CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ +#variable_conflict use_variable +DECLARE + args jsonb := j->'args'; + arg jsonb; + op text := lower(j->>'op'); + cql2op RECORD; + literal text; + _wrapper text; + leftarg text; + rightarg text; +BEGIN + IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN + RETURN NULL; + END IF; + RAISE NOTICE 'CQL2_QUERY: %', j; + IF j ? 'filter' THEN + RETURN cql2_query(j->'filter'); + END IF; + + IF j ? 'upper' THEN + RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); + END IF; + + IF j ? 'lower' THEN + RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); + END IF; + + -- Temporal Query + IF op ilike 't_%' or op = 'anyinteracts' THEN + RETURN temporal_op_query(op, args); + END IF; + + -- If property is a timestamp convert it to text to use with + -- general operators + IF j ? 'timestamp' THEN + RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); + END IF; + IF j ? 'interval' THEN + RAISE EXCEPTION 'Please use temporal operators when using intervals.'; + RETURN NONE; + END IF; + + -- Spatial Query + IF op ilike 's_%' or op = 'intersects' THEN + RETURN spatial_op_query(op, args); + END IF; + + IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN + IF args->0 ? 'property' THEN + leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); + END IF; + IF args->1 ? 'property' THEN + rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); + END IF; + RETURN FORMAT( + '%s %s %s', + COALESCE(leftarg, quote_literal(to_text_array(args->0))), + CASE op + WHEN 'a_equals' THEN '=' + WHEN 'a_contains' THEN '@>' + WHEN 'a_contained_by' THEN '<@' + WHEN 'a_overlaps' THEN '&&' + END, + COALESCE(rightarg, quote_literal(to_text_array(args->1))) + ); + END IF; + + IF op = 'in' THEN + RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; + args := jsonb_build_array(args->0) || (args->1); + RAISE NOTICE 'IN2 : %', args; + END IF; + + + + IF op = 'between' THEN + args = jsonb_build_array( + args->0, + args->1->0, + args->1->1 + ); + END IF; + + -- Make sure that args is an array and run cql2_query on + -- each element of the array + RAISE NOTICE 'ARGS PRE: %', args; + IF j ? 'args' THEN + IF jsonb_typeof(args) != 'array' THEN + args := jsonb_build_array(args); + END IF; + + IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN + wrapper := NULL; + ELSE + -- if any of the arguments are a property, try to get the property_wrapper + FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP + RAISE NOTICE 'Arg: %', arg; + wrapper := (queryable(arg->>'property')).nulled_wrapper; + RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; + IF wrapper IS NOT NULL THEN + EXIT; + END IF; + END LOOP; + + -- if the property was not in queryables, see if any args were numbers + IF + wrapper IS NULL + AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') + THEN + wrapper := 'to_float'; + END IF; + wrapper := coalesce(wrapper, 'to_text'); + END IF; + + SELECT jsonb_agg(cql2_query(a, wrapper)) + INTO args + FROM jsonb_array_elements(args) a; + END IF; + RAISE NOTICE 'ARGS: %', args; + + IF op IN ('and', 'or') THEN + RETURN + format( + '(%s)', + array_to_string(to_text_array(args), format(' %s ', upper(op))) + ); + END IF; + + IF op = 'in' THEN + RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); + RETURN format( + '%s IN (%s)', + to_text(args->0), + array_to_string((to_text_array(args))[2:], ',') + ); + END IF; + + -- Look up template from cql2_ops + IF j ? 'op' THEN + SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; + IF FOUND THEN + -- If specific index set in queryables for a property cast other arguments to that type + + RETURN format( + cql2op.template, + VARIADIC (to_text_array(args)) + ); + ELSE + RAISE EXCEPTION 'Operator % Not Supported.', op; + END IF; + END IF; + + + IF wrapper IS NOT NULL THEN + RAISE NOTICE 'Wrapping % with %', j, wrapper; + IF j ? 'property' THEN + RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); + ELSE + RETURN format('%I(%L)', wrapper, j); + END IF; + ELSIF j ? 'property' THEN + RETURN quote_ident(j->>'property'); + END IF; + + RETURN quote_literal(to_text(j)); +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION paging_dtrange( + j jsonb +) RETURNS tstzrange AS $$ +DECLARE + op text; + filter jsonb := j->'filter'; + dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); + sdate timestamptz := '-infinity'::timestamptz; + edate timestamptz := 'infinity'::timestamptz; + jpitem jsonb; +BEGIN + + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP + op := lower(jpitem->>'op'); + dtrange := parse_dtrange(jpitem->'args'->1); + IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN + sdate := greatest(sdate,'-infinity'); + edate := least(edate, upper(dtrange)); + ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN + edate := least(edate, 'infinity'); + sdate := greatest(sdate, lower(dtrange)); + ELSIF op IN ('=', 'eq') THEN + edate := least(edate, upper(dtrange)); + sdate := greatest(sdate, lower(dtrange)); + END IF; + RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; + END LOOP; + END IF; + IF sdate > edate THEN + RETURN 'empty'::tstzrange; + END IF; + RETURN tstzrange(sdate,edate, '[]'); +END; +$$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; + +CREATE OR REPLACE FUNCTION paging_collections( + IN j jsonb +) RETURNS text[] AS $$ +DECLARE + filter jsonb := j->'filter'; + jpitem jsonb; + op text; + args jsonb; + arg jsonb; + collections text[]; +BEGIN + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + END IF; + IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN + FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP + RAISE NOTICE 'JPITEM: %', jpitem; + op := jpitem->>'op'; + args := jpitem->'args'; + IF op IN ('=', 'eq', 'in') THEN + FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP + IF jsonb_typeof(arg) IN ('string', 'array') THEN + RAISE NOTICE 'arg: %, collections: %', arg, collections; + IF collections IS NULL OR collections = '{}'::text[] THEN + collections := to_text_array(arg); + ELSE + collections := array_intersection(collections, to_text_array(arg)); + END IF; + END IF; + END LOOP; + END IF; + END LOOP; + END IF; + IF collections = '{}'::text[] THEN + RETURN NULL; + END IF; + RETURN collections; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; +CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ + SELECT jsonb_build_object( + 'type', 'Feature', + 'stac_version', content->'stac_version', + 'assets', content->'item_assets', + 'collection', content->'id' + ); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE TABLE IF NOT EXISTS collections ( + key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, + content JSONB NOT NULL, + base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, + partition_trunc text CHECK (partition_trunc IN ('year', 'month')) +); + + + +CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ + SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ +DECLARE + out collections%ROWTYPE; +BEGIN + UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ON CONFLICT (id) DO + UPDATE + SET content=EXCLUDED.content + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ +DECLARE + out collections%ROWTYPE; +BEGIN + DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ + SELECT content FROM collections + WHERE id=$1 + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ + SELECT jsonb_agg(content) FROM collections; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; +CREATE TABLE items ( + id text NOT NULL, + geometry geometry NOT NULL, + collection text NOT NULL, + datetime timestamptz NOT NULL, + end_datetime timestamptz NOT NULL, + content JSONB NOT NULL +) +PARTITION BY LIST (collection) +; + +CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); +CREATE INDEX "geometry_idx" ON items USING GIST (geometry); + +CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; + +ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; + +CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ +DECLARE + p text; + t timestamptz := clock_timestamp(); +BEGIN + RAISE NOTICE 'Updating partition stats %', t; + FOR p IN SELECT DISTINCT partition + FROM newdata n JOIN partition_sys_meta p + ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) + LOOP + PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); + END LOOP; + RAISE NOTICE 't: % %', t, clock_timestamp() - t; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + +CREATE OR REPLACE TRIGGER items_after_insert_trigger +AFTER INSERT ON items +REFERENCING NEW TABLE AS newdata +FOR EACH STATEMENT +EXECUTE FUNCTION partition_after_triggerfunc(); + +CREATE OR REPLACE TRIGGER items_after_update_trigger +AFTER DELETE ON items +REFERENCING OLD TABLE AS newdata +FOR EACH STATEMENT +EXECUTE FUNCTION partition_after_triggerfunc(); + +CREATE OR REPLACE TRIGGER items_after_delete_trigger +AFTER UPDATE ON items +REFERENCING NEW TABLE AS newdata +FOR EACH STATEMENT +EXECUTE FUNCTION partition_after_triggerfunc(); + + +CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ + SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ + SELECT + content->>'id' as id, + stac_geom(content) as geometry, + content->>'collection' as collection, + stac_datetime(content) as datetime, + stac_end_datetime(content) as end_datetime, + content_slim(content) as content + ; +$$ LANGUAGE SQL STABLE; + +CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ +DECLARE + includes jsonb := fields->'include'; + excludes jsonb := fields->'exclude'; +BEGIN + IF f IS NULL THEN + RETURN NULL; + END IF; + + + IF + jsonb_typeof(excludes) = 'array' + AND jsonb_array_length(excludes)>0 + AND excludes ? f + THEN + RETURN FALSE; + END IF; + + IF + ( + jsonb_typeof(includes) = 'array' + AND jsonb_array_length(includes) > 0 + AND includes ? f + ) OR + ( + includes IS NULL + OR jsonb_typeof(includes) = 'null' + OR jsonb_array_length(includes) = 0 + ) + THEN + RETURN TRUE; + END IF; + + RETURN FALSE; +END; +$$ LANGUAGE PLPGSQL IMMUTABLE; + +DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); +CREATE OR REPLACE FUNCTION content_hydrate( + _item jsonb, + _base_item jsonb, + fields jsonb DEFAULT '{}'::jsonb +) RETURNS jsonb AS $$ + SELECT merge_jsonb( + jsonb_fields(_item, fields), + jsonb_fields(_base_item, fields) + ); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + + +CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ +DECLARE + geom jsonb; + bbox jsonb; + output jsonb; + content jsonb; + base_item jsonb := _collection.base_item; +BEGIN + IF include_field('geometry', fields) THEN + geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; + END IF; + output := content_hydrate( + jsonb_build_object( + 'id', _item.id, + 'geometry', geom, + 'collection', _item.collection, + 'type', 'Feature' + ) || _item.content, + _collection.base_item, + fields + ); + + RETURN output; +END; +$$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_nonhydrated( + _item items, + fields jsonb DEFAULT '{}'::jsonb +) RETURNS jsonb AS $$ +DECLARE + geom jsonb; + bbox jsonb; + output jsonb; +BEGIN + IF include_field('geometry', fields) THEN + geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; + END IF; + output := jsonb_build_object( + 'id', _item.id, + 'geometry', geom, + 'collection', _item.collection, + 'type', 'Feature' + ) || _item.content; + RETURN output; +END; +$$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ + SELECT content_hydrate( + _item, + (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), + fields + ); +$$ LANGUAGE SQL STABLE; + + +CREATE UNLOGGED TABLE items_staging ( + content JSONB NOT NULL +); +CREATE UNLOGGED TABLE items_staging_ignore ( + content JSONB NOT NULL +); +CREATE UNLOGGED TABLE items_staging_upsert ( + content JSONB NOT NULL +); + +CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ +DECLARE + p record; + _partitions text[]; + part text; + ts timestamptz := clock_timestamp(); +BEGIN + RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; + + FOR part IN WITH t AS ( + SELECT + n.content->>'collection' as collection, + stac_daterange(n.content->'properties') as dtr, + partition_trunc + FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) + ), p AS ( + SELECT + collection, + COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, + tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, + tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange + FROM t + GROUP BY 1,2 + ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP + RAISE NOTICE 'Partition %', part; + END LOOP; + + RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; + IF TG_TABLE_NAME = 'items_staging' THEN + INSERT INTO items + SELECT + (content_dehydrate(content)).* + FROM newdata; + RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; + DELETE FROM items_staging; + ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN + INSERT INTO items + SELECT + (content_dehydrate(content)).* + FROM newdata + ON CONFLICT DO NOTHING; + RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; + DELETE FROM items_staging_ignore; + ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN + WITH staging_formatted AS ( + SELECT (content_dehydrate(content)).* FROM newdata + ), deletes AS ( + DELETE FROM items i USING staging_formatted s + WHERE + i.id = s.id + AND i.collection = s.collection + AND i IS DISTINCT FROM s + RETURNING i.id, i.collection + ) + INSERT INTO items + SELECT s.* FROM + staging_formatted s + ON CONFLICT DO NOTHING; + RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; + DELETE FROM items_staging_upsert; + END IF; + RAISE NOTICE 'Done. %', clock_timestamp() - ts; + + RETURN NULL; + +END; +$$ LANGUAGE PLPGSQL; + + +CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + +CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + +CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata + FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); + + +CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS +$$ +DECLARE + i items%ROWTYPE; +BEGIN + SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; + RETURN i; +END; +$$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ + SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); +$$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ +DECLARE +out items%ROWTYPE; +BEGIN + DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL; + +--/* +CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging (content) VALUES (data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ +DECLARE + old items %ROWTYPE; + out items%ROWTYPE; +BEGIN + PERFORM delete_item(content->>'id', content->>'collection'); + PERFORM create_item(content); +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging_upsert (content) VALUES (data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging (content) + SELECT * FROM jsonb_array_elements(data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ + INSERT INTO items_staging_upsert (content) + SELECT * FROM jsonb_array_elements(data); +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ + SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb + FROM items WHERE collection=$1; + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ + SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) + FROM items WHERE collection=$1; +; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ +UPDATE collections SET + content = content || + jsonb_build_object( + 'extent', jsonb_build_object( + 'spatial', jsonb_build_object( + 'bbox', collection_bbox(collections.id) + ), + 'temporal', jsonb_build_object( + 'interval', collection_temporal_extent(collections.id) + ) + ) + ) +; +$$ LANGUAGE SQL; +CREATE TABLE partition_stats ( + partition text PRIMARY KEY, + dtrange tstzrange, + edtrange tstzrange, + spatial geometry, + last_updated timestamptz, + keys text[] +) WITH (FILLFACTOR=90); + +CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); + + +CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ + WITH t AS ( + SELECT regexp_matches( + expr, + E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' + ) AS m + ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; + +CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ +DECLARE + expr text := pg_get_constraintdef(coid); + matches timestamptz[]; +BEGIN + IF expr LIKE '%NULL%' THEN + dt := tstzrange(null::timestamptz, null::timestamptz); + edt := tstzrange(null::timestamptz, null::timestamptz); + RETURN; + END IF; + WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) + SELECT array_agg(f::timestamptz) INTO matches FROM f; + IF cardinality(matches) = 4 THEN + dt := tstzrange(matches[1], matches[2],'[]'); + edt := tstzrange(matches[3], matches[4], '[]'); + RETURN; + ELSIF cardinality(matches) = 2 THEN + edt := tstzrange(matches[1], matches[2],'[]'); + RETURN; + END IF; + RETURN; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + +CREATE OR REPLACE VIEW partition_sys_meta AS +SELECT + relid::text as partition, + replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) + ELSE pg_get_expr(parent.relpartbound, parent.oid) + END, 'FOR VALUES IN (''',''), ''')','') AS collection, + level, + c.reltuples, + c.relhastriggers, + COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, + COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, + COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange +FROM + pg_partition_tree('items') + JOIN pg_class c ON (relid::regclass = c.oid) + JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) + LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') +WHERE isleaf +; + +CREATE VIEW partitions AS +SELECT * FROM partition_sys_meta LEFT JOIN partition_stats USING (partition); + +CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ +DECLARE +BEGIN + PERFORM run_or_queue( + format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) + ); +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ +DECLARE + dtrange tstzrange; + edtrange tstzrange; + cdtrange tstzrange; + cedtrange tstzrange; + extent geometry; + collection text; +BEGIN + RAISE NOTICE 'Updating stats for %.', _partition; + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]') + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange; + extent := st_estimatedextent('pgstac', _partition, 'geometry'); + INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) + SELECT _partition, dtrange, edtrange, extent, now() + ON CONFLICT (partition) DO + UPDATE SET + dtrange=EXCLUDED.dtrange, + edtrange=EXCLUDED.edtrange, + spatial=EXCLUDED.spatial, + last_updated=EXCLUDED.last_updated + ; + SELECT + constraint_dtrange, constraint_edtrange, partitions.collection + INTO cdtrange, cedtrange, collection + FROM partitions WHERE partition = _partition; + + RAISE NOTICE 'Checking if we need to modify constraints.'; + IF + (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) + AND NOT istrigger + THEN + RAISE NOTICE 'Modifying Constraints'; + RAISE NOTICE 'Existing % %', cdtrange, cedtrange; + RAISE NOTICE 'New % %', dtrange, edtrange; + PERFORM drop_table_constraints(_partition); + PERFORM create_table_constraints(_partition, dtrange, edtrange); + END IF; + RAISE NOTICE 'Checking if we need to update collection extents.'; + IF get_setting_bool('update_collection_extent') THEN + RAISE NOTICE 'updating collection extent for %', collection; + PERFORM run_or_queue(format($q$ + UPDATE collections + SET content = jsonb_set_lax( + content, + '{extent}'::text[], + collection_extent(%L), + true, + 'use_json_null' + ) WHERE id=%L + ; + $q$, collection, collection)); + ELSE + RAISE NOTICE 'Not updating collection extent for %', collection; + END IF; +END; +$$ LANGUAGE PLPGSQL STRICT; + + +CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ +DECLARE + c RECORD; + parent_name text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + parent_name := format('_items_%s', c.key); + + + IF c.partition_trunc = 'year' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); + ELSE + partition_name := parent_name; + partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); + END IF; + IF partition_range IS NULL THEN + partition_range := tstzrange( + date_trunc(c.partition_trunc::text, dt), + date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval + ); + END IF; + RETURN; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ +DECLARE + q text; +BEGIN + IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN + RETURN NULL; + END IF; + FOR q IN SELECT FORMAT( + $q$ + ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; + $q$, + t, + conname + ) FROM pg_constraint + WHERE conrelid=t::regclass::oid AND contype='c' + LOOP + EXECUTE q; + END LOOP; + RETURN t; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ +DECLARE + q text; +BEGIN + IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN + RETURN NULL; + END IF; + RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; + IF _dtrange = 'empty' AND _edtrange = 'empty' THEN + q :=format( + $q$ + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + $q$, + t, + format('%s_dt', t), + t, + format('%s_dt', t) + ); + ELSE + q :=format( + $q$ + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK ( + (datetime >= %L) + AND (datetime <= %L) + AND (end_datetime >= %L) + AND (end_datetime <= %L) + ) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + $q$, + t, + format('%s_dt', t), + lower(_dtrange), + upper(_dtrange), + lower(_edtrange), + upper(_edtrange), + t, + format('%s_dt', t) + ); + END IF; + PERFORM run_or_queue(q); + RETURN t; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION check_partition( + _collection text, + _dtrange tstzrange, + _edtrange tstzrange +) RETURNS text AS $$ +DECLARE + c RECORD; + pm RECORD; + _partition_name text; + _partition_dtrange tstzrange; + _constraint_dtrange tstzrange; + _constraint_edtrange tstzrange; + q text; + deferrable_q text; + err_context text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + + IF c.partition_trunc IS NOT NULL THEN + _partition_dtrange := tstzrange( + date_trunc(c.partition_trunc, lower(_dtrange)), + date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, + '[)' + ); + ELSE + _partition_dtrange := '[-infinity, infinity]'::tstzrange; + END IF; + + IF NOT _partition_dtrange @> _dtrange THEN + RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; + END IF; + + + IF c.partition_trunc = 'year' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); + ELSE + _partition_name := format('_items_%s', c.key); + END IF; + + SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; + IF FOUND THEN + RAISE NOTICE '% % %', _edtrange, _dtrange, pm; + _constraint_edtrange := + tstzrange( + least( + lower(_edtrange), + nullif(lower(pm.constraint_edtrange), '-infinity') + ), + greatest( + upper(_edtrange), + nullif(upper(pm.constraint_edtrange), 'infinity') + ), + '[]' + ); + _constraint_dtrange := + tstzrange( + least( + lower(_dtrange), + nullif(lower(pm.constraint_dtrange), '-infinity') + ), + greatest( + upper(_dtrange), + nullif(upper(pm.constraint_dtrange), 'infinity') + ), + '[]' + ); + + IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN + RETURN pm.partition; + ELSE + PERFORM drop_table_constraints(_partition_name); + END IF; + ELSE + _constraint_edtrange := _edtrange; + _constraint_dtrange := _dtrange; + END IF; + RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; + IF c.partition_trunc IS NULL THEN + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + _partition_name, + _collection, + concat(_partition_name,'_pk'), + _partition_name + ); + ELSE + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); + CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + format('_items_%s', c.key), + _collection, + _partition_name, + format('_items_%s', c.key), + lower(_partition_dtrange), + upper(_partition_dtrange), + format('%s_pk', _partition_name), + _partition_name + ); + END IF; + + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', _partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); + PERFORM maintain_partitions(_partition_name); + PERFORM update_partition_stats_q(_partition_name, true); + RETURN _partition_name; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ +DECLARE + c RECORD; + q text; + from_trunc text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + IF triggered THEN + RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; + ELSE + RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; + IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN + RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; + RETURN _collection; + END IF; + END IF; + + IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN + EXECUTE format( + $q$ + CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; + DROP TABLE IF EXISTS %I CASCADE; + WITH p AS ( + SELECT + collection, + CASE WHEN %L IS NULL THEN '-infinity'::timestamptz + ELSE date_trunc(%L, datetime) + END as d, + tstzrange(min(datetime),max(datetime),'[]') as dtrange, + tstzrange(min(datetime),max(datetime),'[]') as edtrange + FROM changepartitionstaging + GROUP BY 1,2 + ) SELECT check_partition(collection, dtrange, edtrange) FROM p; + INSERT INTO items SELECT * FROM changepartitionstaging; + DROP TABLE changepartitionstaging; + $q$, + concat('_items_', c.key), + concat('_items_', c.key), + c.partition_trunc, + c.partition_trunc + ); + END IF; + RETURN _collection; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + q text; + partition_name text := format('_items_%s', NEW.key); + partition_exists boolean := false; + partition_empty boolean := true; + err_context text; + loadtemp boolean := FALSE; +BEGIN + RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN + PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); + END IF; + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE TRIGGER collections_trigger AFTER +INSERT +OR +UPDATE ON collections +FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); +CREATE OR REPLACE VIEW partition_steps AS +SELECT + partition as name, + date_trunc('month',lower(partition_dtrange)) as sdate, + date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate + FROM partitions WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange + ORDER BY dtrange ASC +; + +CREATE OR REPLACE FUNCTION chunker( + IN _where text, + OUT s timestamptz, + OUT e timestamptz +) RETURNS SETOF RECORD AS $$ +DECLARE + explain jsonb; +BEGIN + IF _where IS NULL THEN + _where := ' TRUE '; + END IF; + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) + INTO explain; + + RETURN QUERY + WITH t AS ( + SELECT j->>0 as p FROM + jsonb_path_query( + explain, + 'strict $.**."Relation Name" ? (@ != null)' + ) j + ), + parts AS ( + SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) + ), + times AS ( + SELECT sdate FROM parts + UNION + SELECT edate FROM parts + ), + uniq AS ( + SELECT DISTINCT sdate FROM times ORDER BY sdate + ), + last AS ( + SELECT sdate, lead(sdate, 1) over () as edate FROM uniq + ) + SELECT sdate, edate FROM last WHERE edate IS NOT NULL; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION partition_queries( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN partitions text[] DEFAULT NULL +) RETURNS SETOF text AS $$ +DECLARE + query text; + sdate timestamptz; + edate timestamptz; +BEGIN +IF _where IS NULL OR trim(_where) = '' THEN + _where = ' TRUE '; +END IF; +RAISE NOTICE 'Getting chunks for % %', _where, _orderby; +IF _orderby ILIKE 'datetime d%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, + _where, + _orderby + ); + END LOOP; +ELSIF _orderby ILIKE 'datetime a%' THEN + FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP + RETURN NEXT format($q$ + SELECT * FROM items + WHERE + datetime >= %L AND datetime < %L + AND (%s) + ORDER BY %s + $q$, + sdate, + edate, + _where, + _orderby + ); + END LOOP; +ELSE + query := format($q$ + SELECT * FROM items + WHERE %s + ORDER BY %s + $q$, _where, _orderby + ); + + RETURN NEXT query; + RETURN; +END IF; + +RETURN; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION partition_query_view( + IN _where text DEFAULT 'TRUE', + IN _orderby text DEFAULT 'datetime DESC, id DESC', + IN _limit int DEFAULT 10 +) RETURNS text AS $$ + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total LIMIT %s + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ), + _limit + )) + ELSE NULL + END FROM p; +$$ LANGUAGE SQL IMMUTABLE; + + + + +CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ +DECLARE + where_segments text[]; + _where text; + dtrange tstzrange; + collections text[]; + geom geometry; + sdate timestamptz; + edate timestamptz; + filterlang text; + filter jsonb := j->'filter'; +BEGIN + IF j ? 'ids' THEN + where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); + END IF; + + IF j ? 'collections' THEN + collections := to_text_array(j->'collections'); + where_segments := where_segments || format('collection = ANY (%L) ', collections); + END IF; + + IF j ? 'datetime' THEN + dtrange := parse_dtrange(j->'datetime'); + sdate := lower(dtrange); + edate := upper(dtrange); + + where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', + edate, + sdate + ); + END IF; + + geom := stac_geom(j); + IF geom IS NOT NULL THEN + where_segments := where_segments || format('st_intersects(geometry, %L)',geom); + END IF; + + filterlang := COALESCE( + j->>'filter-lang', + get_setting('default_filter_lang', j->'conf') + ); + IF NOT filter @? '$.**.op' THEN + filterlang := 'cql-json'; + END IF; + + IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN + RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; + END IF; + + IF j ? 'query' AND j ? 'filter' THEN + RAISE EXCEPTION 'Can only use either query or filter at one time.'; + END IF; + + IF j ? 'query' THEN + filter := query_to_cql2(j->'query'); + ELSIF filterlang = 'cql-json' THEN + filter := cql1_to_cql2(filter); + END IF; + RAISE NOTICE 'FILTER: %', filter; + where_segments := where_segments || cql2_query(filter); + IF cardinality(where_segments) < 1 THEN + RETURN ' TRUE '; + END IF; + + _where := array_to_string(array_remove(where_segments, NULL), ' AND '); + + IF _where IS NULL OR BTRIM(_where) = '' THEN + RETURN ' TRUE '; + END IF; + RETURN _where; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN NOT reverse THEN d + WHEN d = 'ASC' THEN 'DESC' + WHEN d = 'DESC' THEN 'ASC' + END + FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ + WITH t AS ( + SELECT COALESCE(upper(_dir), 'ASC') as d + ) SELECT + CASE + WHEN d = 'ASC' AND prev THEN '<=' + WHEN d = 'DESC' AND prev THEN '>=' + WHEN d = 'ASC' THEN '>=' + WHEN d = 'DESC' THEN '<=' + END + FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION sort_sqlorderby( + _search jsonb DEFAULT NULL, + reverse boolean DEFAULT FALSE +) RETURNS text AS $$ + WITH sortby AS ( + SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort + ), withid AS ( + SELECT CASE + WHEN sort @? '$[*] ? (@.field == "id")' THEN sort + ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb + END as sort + FROM sortby + ), withid_rows AS ( + SELECT jsonb_array_elements(sort) as value FROM withid + ),sorts AS ( + SELECT + coalesce( + -- field_orderby((items_path(value->>'field')).path_txt), + (queryable(value->>'field')).expression + ) as key, + parse_sort_dir(value->>'direction', reverse) as dir + FROM withid_rows + ) + SELECT array_to_string( + array_agg(concat(key, ' ', dir)), + ', ' + ) FROM sorts; +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ + SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION get_token_val_str( + _field text, + _item items +) RETURNS text AS $$ +DECLARE +literal text; +BEGIN +RAISE NOTICE '% %', _field, _item; +CREATE TEMP TABLE _token_item ON COMMIT DROP AS SELECT (_item).*; +EXECUTE format($q$ SELECT quote_literal(%s) FROM _token_item $q$, _field) INTO literal; +DROP TABLE IF EXISTS _token_item; +RETURN literal; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ +DECLARE + token_id text; + filters text[] := '{}'::text[]; + prev boolean := TRUE; + field text; + dir text; + sort record; + orfilters text[] := '{}'::text[]; + andfilters text[] := '{}'::text[]; + output text; + token_where text; + token_item items%ROWTYPE; +BEGIN + RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; + -- If no token provided return NULL + IF token_rec IS NULL THEN + IF NOT (_search ? 'token' AND + ( + (_search->>'token' ILIKE 'prev:%') + OR + (_search->>'token' ILIKE 'next:%') + ) + ) THEN + RETURN NULL; + END IF; + prev := (_search->>'token' ILIKE 'prev:%'); + token_id := substr(_search->>'token', 6); + SELECT to_jsonb(items) INTO token_rec + FROM items WHERE id=token_id; + END IF; + RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; + + + RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; + token_item := jsonb_populate_record(null::items, token_rec); + RAISE NOTICE 'TOKEN ITEM ----- %', token_item; + + + CREATE TEMP TABLE sorts ( + _row int GENERATED ALWAYS AS IDENTITY NOT NULL, + _field text PRIMARY KEY, + _dir text NOT NULL, + _val text + ) ON COMMIT DROP; + + -- Make sure we only have distinct columns to sort with taking the first one we get + INSERT INTO sorts (_field, _dir) + SELECT + (queryable(value->>'field')).expression, + get_sort_dir(value) + FROM + jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) + ON CONFLICT DO NOTHING + ; + RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); + -- Get the first sort direction provided. As the id is a primary key, if there are any + -- sorts after id they won't do anything, so make sure that id is the last sort item. + SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; + IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN + DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); + ELSE + INSERT INTO sorts (_field, _dir) VALUES ('id', dir); + END IF; + + -- Add value from looked up item to the sorts table + UPDATE sorts SET _val=get_token_val_str(_field, token_item); + + -- Check if all sorts are the same direction and use row comparison + -- to filter + RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); + + FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP + RAISE NOTICE 'SORT: %', sort; + IF sort._row = 1 THEN + IF sort._val IS NULL THEN + orfilters := orfilters || format('(%s IS NOT NULL)', sort._field); + ELSE + orfilters := orfilters || format('(%s %s %s)', + sort._field, + CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, + sort._val + ); + END IF; + ELSE + IF sort._val IS NULL THEN + orfilters := orfilters || format('(%s AND %s IS NOT NULL)', + array_to_string(andfilters, ' AND '), sort._field); + ELSE + orfilters := orfilters || format('(%s AND %s %s %s)', + array_to_string(andfilters, ' AND '), + sort._field, + CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, + sort._val + ); + END IF; + END IF; + IF sort._val IS NULL THEN + andfilters := andfilters || format('%s IS NULL', + sort._field + ); + ELSE + andfilters := andfilters || format('%s = %s', + sort._field, + sort._val + ); + END IF; + END LOOP; + output := array_to_string(orfilters, ' OR '); + + DROP TABLE IF EXISTS sorts; + token_where := concat('(',coalesce(output,'true'),')'); + IF trim(token_where) = '' THEN + token_where := NULL; + END IF; + RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; + RETURN token_where; + END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ + SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ + SELECT md5(concat(search_tohash($1)::text,$2::text)); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE TABLE IF NOT EXISTS searches( + hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, + search jsonb NOT NULL, + _where text, + orderby text, + lastused timestamptz DEFAULT now(), + usecount bigint DEFAULT 0, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL +); + +CREATE TABLE IF NOT EXISTS search_wheres( + id bigint generated always as identity primary key, + _where text NOT NULL, + lastused timestamptz DEFAULT now(), + usecount bigint DEFAULT 0, + statslastupdated timestamptz, + estimated_count bigint, + estimated_cost float, + time_to_estimate float, + total_count bigint, + time_to_count float, + partitions text[] +); + +CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); +CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); + +CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ +DECLARE + t timestamptz; + i interval; + explain_json jsonb; + partitions text[]; + sw search_wheres%ROWTYPE; + inwhere_hash text := md5(inwhere); + _context text := lower(context(conf)); + _stats_ttl interval := context_stats_ttl(conf); + _estimated_cost float := context_estimated_cost(conf); + _estimated_count int := context_estimated_count(conf); +BEGIN + IF _context = 'off' THEN + sw._where := inwhere; + return sw; + END IF; + + SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; + + -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired + IF NOT updatestats THEN + RAISE NOTICE 'Checking if update is needed for: % .', inwhere; + RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; + RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; + RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; + IF + sw.statslastupdated IS NULL + OR (now() - sw.statslastupdated) > _stats_ttl + OR (context(conf) != 'off' AND sw.total_count IS NULL) + THEN + updatestats := TRUE; + END IF; + END IF; + + sw._where := inwhere; + sw.lastused := now(); + sw.usecount := coalesce(sw.usecount,0) + 1; + + IF NOT updatestats THEN + UPDATE search_wheres SET + lastused = sw.lastused, + usecount = sw.usecount + WHERE md5(_where) = inwhere_hash + RETURNING * INTO sw + ; + RETURN sw; + END IF; + + -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query + t := clock_timestamp(); + EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) + INTO explain_json; + RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; + i := clock_timestamp() - t; + + sw.statslastupdated := now(); + sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; + sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; + sw.time_to_estimate := extract(epoch from i); + + RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; + RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; + + -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough + IF + _context = 'on' + OR + ( _context = 'auto' AND + ( + sw.estimated_count < _estimated_count + AND + sw.estimated_cost < _estimated_cost + ) + ) + THEN + t := clock_timestamp(); + RAISE NOTICE 'Calculating actual count...'; + EXECUTE format( + 'SELECT count(*) FROM items WHERE %s', + inwhere + ) INTO sw.total_count; + i := clock_timestamp() - t; + RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; + sw.time_to_count := extract(epoch FROM i); + ELSE + sw.total_count := NULL; + sw.time_to_count := NULL; + END IF; + + + INSERT INTO search_wheres + (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) + SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count + ON CONFLICT ((md5(_where))) + DO UPDATE + SET + lastused = sw.lastused, + usecount = sw.usecount, + statslastupdated = sw.statslastupdated, + estimated_count = sw.estimated_count, + estimated_cost = sw.estimated_cost, + time_to_estimate = sw.time_to_estimate, + total_count = sw.total_count, + time_to_count = sw.time_to_count + ; + RETURN sw; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION search_query( + _search jsonb = '{}'::jsonb, + updatestats boolean = false, + _metadata jsonb = '{}'::jsonb +) RETURNS searches AS $$ +DECLARE + search searches%ROWTYPE; + pexplain jsonb; + t timestamptz; + i interval; +BEGIN + SELECT * INTO search FROM searches + WHERE hash=search_hash(_search, _metadata) FOR UPDATE; + + -- Calculate the where clause if not already calculated + IF search._where IS NULL THEN + search._where := stac_search_to_where(_search); + END IF; + + -- Calculate the order by clause if not already calculated + IF search.orderby IS NULL THEN + search.orderby := sort_sqlorderby(_search); + END IF; + + PERFORM where_stats(search._where, updatestats, _search->'conf'); + + search.lastused := now(); + search.usecount := coalesce(search.usecount, 0) + 1; + INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) + VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) + ON CONFLICT (hash) DO + UPDATE SET + _where = EXCLUDED._where, + orderby = EXCLUDED.orderby, + lastused = EXCLUDED.lastused, + usecount = EXCLUDED.usecount, + metadata = EXCLUDED.metadata + RETURNING * INTO search + ; + RETURN search; + +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ +DECLARE + searches searches%ROWTYPE; + _where text; + token_where text; + full_where text; + orderby text; + query text; + token_type text := substr(_search->>'token',1,4); + _limit int := coalesce((_search->>'limit')::int, 10); + curs refcursor; + cntr int := 0; + iter_record items%ROWTYPE; + first_record jsonb; + first_item items%ROWTYPE; + last_item items%ROWTYPE; + last_record jsonb; + out_records jsonb := '[]'::jsonb; + prev_query text; + next text; + prev_id text; + has_next boolean := false; + has_prev boolean := false; + prev text; + total_count bigint; + context jsonb; + collection jsonb; + includes text[]; + excludes text[]; + exit_flag boolean := FALSE; + batches int := 0; + timer timestamptz := clock_timestamp(); + pstart timestamptz; + pend timestamptz; + pcurs refcursor; + search_where search_wheres%ROWTYPE; + id text; +BEGIN +CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; +-- if ids is set, short circuit and just use direct ids query for each id +-- skip any paging or caching +-- hard codes ordering in the same order as the array of ids +IF _search ? 'ids' THEN + INSERT INTO results (content) + SELECT + CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN + content_nonhydrated(items, _search->'fields') + ELSE + content_hydrate(items, _search->'fields') + END + FROM items WHERE + items.id = ANY(to_text_array(_search->'ids')) + AND + CASE WHEN _search ? 'collections' THEN + items.collection = ANY(to_text_array(_search->'collections')) + ELSE TRUE + END + ORDER BY items.datetime desc, items.id desc + ; + SELECT INTO total_count count(*) FROM results; +ELSE + searches := search_query(_search); + _where := searches._where; + orderby := searches.orderby; + search_where := where_stats(_where); + total_count := coalesce(search_where.total_count, search_where.estimated_count); + + IF token_type='prev' THEN + token_where := get_token_filter(_search, null::jsonb); + orderby := sort_sqlorderby(_search, TRUE); + END IF; + IF token_type='next' THEN + token_where := get_token_filter(_search, null::jsonb); + END IF; + + full_where := concat_ws(' AND ', _where, token_where); + RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; + timer := clock_timestamp(); + + FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP + timer := clock_timestamp(); + query := format('%s LIMIT %s', query, _limit + 1); + RAISE NOTICE 'Partition Query: %', query; + batches := batches + 1; + -- curs = create_cursor(query); + RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); + OPEN curs FOR EXECUTE query; + LOOP + FETCH curs into iter_record; + EXIT WHEN NOT FOUND; + cntr := cntr + 1; + + IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN + last_record := content_nonhydrated(iter_record, _search->'fields'); + ELSE + last_record := content_hydrate(iter_record, _search->'fields'); + END IF; + last_item := iter_record; + IF cntr = 1 THEN + first_item := last_item; + first_record := last_record; + END IF; + IF cntr <= _limit THEN + INSERT INTO results (content) VALUES (last_record); + ELSIF cntr > _limit THEN + has_next := true; + exit_flag := true; + EXIT; + END IF; + END LOOP; + CLOSE curs; + RAISE NOTICE 'Query took %.', clock_timestamp()-timer; + timer := clock_timestamp(); + EXIT WHEN exit_flag; + END LOOP; + RAISE NOTICE 'Scanned through % partitions.', batches; +END IF; + +WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) +SELECT jsonb_agg(content) INTO out_records FROM ordered; + +DROP TABLE results; + + +-- Flip things around if this was the result of a prev token query +IF token_type='prev' THEN + out_records := flip_jsonb_array(out_records); + first_item := last_item; + first_record := last_record; +END IF; + +-- If this query has a token, see if there is data before the first record +IF _search ? 'token' THEN + prev_query := format( + 'SELECT 1 FROM items WHERE %s LIMIT 1', + concat_ws( + ' AND ', + _where, + trim(get_token_filter(_search, to_jsonb(first_item))) + ) + ); + RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; + EXECUTE prev_query INTO has_prev; + IF FOUND and has_prev IS NOT NULL THEN + RAISE NOTICE 'Query results from prev query: %', has_prev; + has_prev := TRUE; + END IF; +END IF; +has_prev := COALESCE(has_prev, FALSE); + +IF has_prev THEN + prev := out_records->0->>'id'; +END IF; +IF has_next OR token_type='prev' THEN + next := out_records->-1->>'id'; +END IF; + +IF context(_search->'conf') != 'off' THEN + context := jsonb_strip_nulls(jsonb_build_object( + 'limit', _limit, + 'matched', total_count, + 'returned', coalesce(jsonb_array_length(out_records), 0) + )); +ELSE + context := jsonb_strip_nulls(jsonb_build_object( + 'limit', _limit, + 'returned', coalesce(jsonb_array_length(out_records), 0) + )); +END IF; + +collection := jsonb_build_object( + 'type', 'FeatureCollection', + 'features', coalesce(out_records, '[]'::jsonb), + 'next', next, + 'prev', prev, + 'context', context +); + +RETURN collection; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; + + +CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ +DECLARE + curs refcursor; + searches searches%ROWTYPE; + _where text; + _orderby text; + q text; + +BEGIN + searches := search_query(_search); + _where := searches._where; + _orderby := searches.orderby; + + OPEN curs FOR + WITH p AS ( + SELECT * FROM partition_queries(_where, _orderby) p + ) + SELECT + CASE WHEN EXISTS (SELECT 1 FROM p) THEN + (SELECT format($q$ + SELECT * FROM ( + %s + ) total + $q$, + string_agg( + format($q$ SELECT * FROM ( %s ) AS sub $q$, p), + ' + UNION ALL + ' + ) + )) + ELSE NULL + END FROM p; + RETURN curs; +END; +$$ LANGUAGE PLPGSQL; +SET SEARCH_PATH TO pgstac, public; + +CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ +WITH t AS ( + SELECT + 20037508.3427892 as merc_max, + -20037508.3427892 as merc_min, + (2 * 20037508.3427892) / (2 ^ zoom) as tile_size +) +SELECT st_makeenvelope( + merc_min + (tile_size * x), + merc_max - (tile_size * (y + 1)), + merc_min + (tile_size * (x + 1)), + merc_max - (tile_size * y), + 3857 +) FROM t; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; + + +CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ +SELECT age(clock_timestamp(), transaction_timestamp()); +$$ LANGUAGE SQL; +SET SEARCH_PATH to pgstac, public; + +DROP FUNCTION IF EXISTS geometrysearch; +CREATE OR REPLACE FUNCTION geometrysearch( + IN geom geometry, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered + IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items +) RETURNS jsonb AS $$ +DECLARE + search searches%ROWTYPE; + curs refcursor; + _where text; + query text; + iter_record items%ROWTYPE; + out_records jsonb := '{}'::jsonb[]; + exit_flag boolean := FALSE; + counter int := 1; + scancounter int := 1; + remaining_limit int := _scanlimit; + tilearea float; + unionedgeom geometry; + clippedgeom geometry; + unionedgeom_area float := 0; + prev_area float := 0; + excludes text[]; + includes text[]; + +BEGIN + DROP TABLE IF EXISTS pgstac_results; + CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; + + -- If skipcovered is true then you will always want to exit when the passed in geometry is full + IF skipcovered THEN + exitwhenfull := TRUE; + END IF; + + SELECT * INTO search FROM searches WHERE hash=queryhash; + + IF NOT FOUND THEN + RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; + END IF; + + tilearea := st_area(geom); + _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); + + + FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP + query := format('%s LIMIT %L', query, remaining_limit); + RAISE NOTICE '%', query; + OPEN curs FOR EXECUTE query; + LOOP + FETCH curs INTO iter_record; + EXIT WHEN NOT FOUND; + IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations + clippedgeom := st_intersection(geom, iter_record.geometry); + + IF unionedgeom IS NULL THEN + unionedgeom := clippedgeom; + ELSE + unionedgeom := st_union(unionedgeom, clippedgeom); + END IF; + + unionedgeom_area := st_area(unionedgeom); + + IF skipcovered AND prev_area = unionedgeom_area THEN + scancounter := scancounter + 1; + CONTINUE; + END IF; + + prev_area := unionedgeom_area; + + RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); + END IF; + RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); + INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); + + IF counter >= _limit + OR scancounter > _scanlimit + OR ftime() > _timelimit + OR (exitwhenfull AND unionedgeom_area >= tilearea) + THEN + exit_flag := TRUE; + EXIT; + END IF; + counter := counter + 1; + scancounter := scancounter + 1; + + END LOOP; + CLOSE curs; + EXIT WHEN exit_flag; + remaining_limit := _scanlimit - scancounter; + END LOOP; + + SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; + + RETURN jsonb_build_object( + 'type', 'FeatureCollection', + 'features', coalesce(out_records, '[]'::jsonb) + ); +END; +$$ LANGUAGE PLPGSQL; + +DROP FUNCTION IF EXISTS geojsonsearch; +CREATE OR REPLACE FUNCTION geojsonsearch( + IN geojson jsonb, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, + IN skipcovered boolean DEFAULT TRUE +) RETURNS jsonb AS $$ + SELECT * FROM geometrysearch( + st_geomfromgeojson(geojson), + queryhash, + fields, + _scanlimit, + _limit, + _timelimit, + exitwhenfull, + skipcovered + ); +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS xyzsearch; +CREATE OR REPLACE FUNCTION xyzsearch( + IN _x int, + IN _y int, + IN _z int, + IN queryhash text, + IN fields jsonb DEFAULT NULL, + IN _scanlimit int DEFAULT 10000, + IN _limit int DEFAULT 100, + IN _timelimit interval DEFAULT '5 seconds'::interval, + IN exitwhenfull boolean DEFAULT TRUE, + IN skipcovered boolean DEFAULT TRUE +) RETURNS jsonb AS $$ + SELECT * FROM geometrysearch( + st_transform(tileenvelope(_z, _x, _y), 4326), + queryhash, + fields, + _scanlimit, + _limit, + _timelimit, + exitwhenfull, + skipcovered + ); +$$ LANGUAGE SQL; + +DROP FUNCTION IF EXISTS analyze_items; +CREATE OR REPLACE PROCEDURE analyze_items() AS $$ +DECLARE + q text; + timeout_ts timestamptz; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + RAISE NOTICE '% % %', clock_timestamp(), timeout_ts, current_setting('statement_timeout', TRUE); + SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q + FROM pg_stat_user_tables + WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; + IF NOT FOUND THEN + EXIT; + END IF; + RAISE NOTICE '%', q; + EXECUTE q; + COMMIT; + RAISE NOTICE '%', queue_timeout(); + END LOOP; +END; +$$ LANGUAGE PLPGSQL; + + +DROP FUNCTION IF EXISTS validate_constraints; +CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ +DECLARE + q text; +BEGIN + FOR q IN + SELECT + FORMAT( + 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', + nsp.nspname, + cls.relname, + con.conname + ) + + FROM pg_constraint AS con + JOIN pg_class AS cls + ON con.conrelid = cls.oid + JOIN pg_namespace AS nsp + ON cls.relnamespace = nsp.oid + WHERE convalidated = FALSE AND contype in ('c','f') + AND nsp.nspname = 'pgstac' + LOOP + RAISE NOTICE '%', q; + PERFORM run_or_queue(q); + COMMIT; + END LOOP; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ +DECLARE + geom_extent geometry; + mind timestamptz; + maxd timestamptz; + extent jsonb; +BEGIN + IF runupdate THEN + PERFORM update_partition_stats_q(partition) + FROM partitions WHERE collection=_collection; + END IF; + SELECT + min(lower(dtrange)), + max(upper(edtrange)), + st_extent(spatial) + INTO + mind, + maxd, + geom_extent + FROM partitions + WHERE collection=_collection; + + IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN + extent := jsonb_build_object( + 'extent', jsonb_build_object( + 'spatial', jsonb_build_object( + 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) + ), + 'temporal', jsonb_build_object( + 'interval', to_jsonb(array[array[mind, maxd]]) + ) + ) + ); + RETURN extent; + END IF; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL; +INSERT INTO queryables (name, definition) VALUES +('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), +('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), +('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') +ON CONFLICT DO NOTHING; + +INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES +('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') +ON CONFLICT DO NOTHING; + + +INSERT INTO pgstac_settings (name, value) VALUES + ('context', 'off'), + ('context_estimated_count', '100000'), + ('context_estimated_cost', '100000'), + ('context_stats_ttl', '1 day'), + ('default_filter_lang', 'cql2-json'), + ('additional_properties', 'true'), + ('use_queue', 'false'), + ('queue_timeout', '10 minutes'), + ('update_collection_extent', 'true') +ON CONFLICT DO NOTHING +; + + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +GRANT ALL ON SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON SCHEMA pgstac to pgstac_admin; + +-- pgstac_read role limited to using function apis +GRANT EXECUTE ON FUNCTION search TO pgstac_read; +GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; +GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; +GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; + +SELECT update_partition_stats_q(partition) FROM partitions; +SELECT set_version('0.7.0'); diff --git a/pgstac.sql b/src/pgstac/pgstac.sql similarity index 65% rename from pgstac.sql rename to src/pgstac/pgstac.sql index a02f376a..dcade13c 100644 --- a/pgstac.sql +++ b/src/pgstac/pgstac.sql @@ -1,14 +1,17 @@ BEGIN; +\i sql/000_idempotent_pre.sql \i sql/001_core.sql \i sql/001a_jsonutils.sql \i sql/001s_stacutils.sql \i sql/002_collections.sql \i sql/002a_queryables.sql \i sql/002b_cql.sql -\i sql/003_items.sql +\i sql/003a_items.sql +\i sql/003b_partitions.sql \i sql/004_search.sql \i sql/005_tileutils.sql \i sql/006_tilesearch.sql -\i sql/998_permissions.sql +\i sql/997_maintenance.sql +\i sql/998_idempotent_post.sql \i sql/999_version.sql COMMIT; diff --git a/src/pgstac/sql/000_idempotent_pre.sql b/src/pgstac/sql/000_idempotent_pre.sql new file mode 100644 index 00000000..759e2118 --- /dev/null +++ b/src/pgstac/sql/000_idempotent_pre.sql @@ -0,0 +1,119 @@ +DO $$ +DECLARE +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN + CREATE EXTENSION IF NOT EXISTS postgis; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN + CREATE EXTENSION IF NOT EXISTS btree_gist; + END IF; +END; +$$ LANGUAGE PLPGSQL; + +DO $$ + BEGIN + CREATE ROLE pgstac_admin; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_read; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + +DO $$ + BEGIN + CREATE ROLE pgstac_ingest; + EXCEPTION WHEN duplicate_object THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END +$$; + + +GRANT pgstac_admin TO current_user; + +-- Function to make sure pgstac_admin is the owner of items +CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ +DECLARE + f RECORD; +BEGIN + FOR f IN ( + SELECT + concat( + oid::regproc::text, + '(', + coalesce(pg_get_function_identity_arguments(oid),''), + ')' + ) AS name, + CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ + FROM pg_proc + WHERE + pronamespace=to_regnamespace('pgstac') + AND proowner != to_regrole('pgstac_admin') + AND proname NOT LIKE 'pg_stat%' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + FOR f IN ( + SELECT + oid::regclass::text as name, + CASE relkind + WHEN 'i' THEN 'INDEX' + WHEN 'I' THEN 'INDEX' + WHEN 'p' THEN 'TABLE' + WHEN 'r' THEN 'TABLE' + WHEN 'v' THEN 'VIEW' + WHEN 'S' THEN 'SEQUENCE' + ELSE NULL + END as typ + FROM pg_class + WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' + ) + LOOP + BEGIN + EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); + EXCEPTION WHEN others THEN + RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; + END; + END LOOP; + RETURN; +END; +$$ LANGUAGE PLPGSQL; +SELECT pgstac_admin_owns(); + +CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; + +GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; +GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; + +ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; +ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; + +GRANT pgstac_read TO pgstac_ingest; +GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; +ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; + +SET ROLE pgstac_admin; + +SET SEARCH_PATH TO pgstac, public; + +DROP FUNCTION IF EXISTS analyze_items; +DROP FUNCTION IF EXISTS validate_constraints; diff --git a/sql/001_core.sql b/src/pgstac/sql/001_core.sql similarity index 70% rename from sql/001_core.sql rename to src/pgstac/sql/001_core.sql index c1e3b74d..b655f329 100644 --- a/sql/001_core.sql +++ b/src/pgstac/sql/001_core.sql @@ -1,38 +1,3 @@ -CREATE EXTENSION IF NOT EXISTS postgis; -CREATE EXTENSION IF NOT EXISTS btree_gist; - -DO $$ - BEGIN - CREATE ROLE pgstac_admin; - CREATE ROLE pgstac_read; - CREATE ROLE pgstac_ingest; - EXCEPTION WHEN duplicate_object THEN - RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; - END -$$; - -GRANT pgstac_admin TO current_user; - -CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; - -ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; -ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; -ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; - -GRANT USAGE ON SCHEMA pgstac to pgstac_read; -ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; -ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; -ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; - -GRANT pgstac_read TO pgstac_ingest; -GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; -ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; -ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; - -SET ROLE pgstac_admin; - -SET SEARCH_PATH TO pgstac, public; - CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, @@ -55,23 +20,35 @@ CREATE TABLE IF NOT EXISTS pgstac_settings ( value text NOT NULL ); -INSERT INTO pgstac_settings (name, value) VALUES - ('context', 'off'), - ('context_estimated_count', '100000'), - ('context_estimated_cost', '100000'), - ('context_stats_ttl', '1 day'), - ('default_filter_lang', 'cql2-json'), - ('additional_properties', 'true') -ON CONFLICT DO NOTHING -; +CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ +DECLARE + retval boolean; +BEGIN + EXECUTE format($q$ + SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) + $q$, + $1 + ) INTO retval; + RETURN retval; +END; +$$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ +SELECT COALESCE( + nullif(conf->>_setting, ''), + nullif(current_setting(concat('pgstac.',_setting), TRUE),''), + nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') +); +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), - (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) -); + (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), + 'FALSE' +)::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ @@ -92,6 +69,20 @@ CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS in SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ + SELECT extract(epoch FROM $1::interval)::text || ' s'; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; + +CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ + SELECT set_config( + 'statement_timeout', + t2s(coalesce( + get_setting('queue_timeout'), + '1h' + )), + false + )::interval; +$$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE @@ -137,6 +128,95 @@ SELECT ARRAY( ); $$ LANGUAGE SQL STRICT IMMUTABLE; +DROP TABLE IF EXISTS query_queue; +CREATE TABLE query_queue ( + query text PRIMARY KEY, + added timestamptz DEFAULT now() +); + +DROP TABLE IF EXISTS query_queue_history; +CREATE TABLE query_queue_history( + query text, + added timestamptz NOT NULL, + finished timestamptz NOT NULL DEFAULT now(), + error text +); + +CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ +DECLARE + qitem query_queue%ROWTYPE; + timeout_ts timestamptz; + error text; + cnt int := 0; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; + IF NOT FOUND THEN + EXIT; + END IF; + cnt := cnt + 1; + BEGIN + RAISE NOTICE 'RUNNING QUERY: %', qitem.query; + EXECUTE qitem.query; + EXCEPTION WHEN others THEN + error := format('%s | %s', SQLERRM, SQLSTATE); + END; + INSERT INTO query_queue_history (query, added, finished, error) + VALUES (qitem.query, qitem.added, clock_timestamp(), error); + COMMIT; + END LOOP; +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ +DECLARE + qitem query_queue%ROWTYPE; + timeout_ts timestamptz; + error text; + cnt int := 0; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; + IF NOT FOUND THEN + RETURN cnt; + END IF; + cnt := cnt + 1; + BEGIN + qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); + RAISE NOTICE 'RUNNING QUERY: %', qitem.query; + + EXECUTE qitem.query; + EXCEPTION WHEN others THEN + error := format('%s | %s', SQLERRM, SQLSTATE); + RAISE WARNING '%', error; + END; + INSERT INTO query_queue_history (query, added, finished, error) + VALUES (qitem.query, qitem.added, clock_timestamp(), error); + END LOOP; + RETURN cnt; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ +DECLARE + use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; +BEGIN + IF get_setting_bool('debug') THEN + RAISE NOTICE '%', query; + END IF; + IF use_queue THEN + INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; + ELSE + EXECUTE query; + END IF; + RETURN; +END; +$$ LANGUAGE PLPGSQL; + + DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ @@ -224,7 +304,10 @@ BEGIN IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE - RAISE NOTICE 'pgstattuple % is installed', settingval; + RAISE NOTICE 'pg_stat_statements % is installed', settingval; + IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN + RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; + END IF; END IF; END; diff --git a/sql/001a_jsonutils.sql b/src/pgstac/sql/001a_jsonutils.sql similarity index 100% rename from sql/001a_jsonutils.sql rename to src/pgstac/sql/001a_jsonutils.sql diff --git a/sql/001s_stacutils.sql b/src/pgstac/sql/001s_stacutils.sql similarity index 97% rename from sql/001s_stacutils.sql rename to src/pgstac/sql/001s_stacutils.sql index ca34de3d..6c1c805e 100644 --- a/sql/001s_stacutils.sql +++ b/src/pgstac/sql/001s_stacutils.sql @@ -57,7 +57,7 @@ CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; -CREATE TABLE stac_extensions( +CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); diff --git a/src/pgstac/sql/002_collections.sql b/src/pgstac/sql/002_collections.sql new file mode 100644 index 00000000..9a7e656f --- /dev/null +++ b/src/pgstac/sql/002_collections.sql @@ -0,0 +1,67 @@ +CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ + SELECT jsonb_build_object( + 'type', 'Feature', + 'stac_version', content->'stac_version', + 'assets', content->'item_assets', + 'collection', content->'id' + ); +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE TABLE IF NOT EXISTS collections ( + key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, + content JSONB NOT NULL, + base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, + partition_trunc text CHECK (partition_trunc IN ('year', 'month')) +); + + + +CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ + SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + + +CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ +DECLARE + out collections%ROWTYPE; +BEGIN + UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + +CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ + INSERT INTO collections (content) + VALUES (data) + ON CONFLICT (id) DO + UPDATE + SET content=EXCLUDED.content + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ +DECLARE + out collections%ROWTYPE; +BEGIN + DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; +END; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ + SELECT content FROM collections + WHERE id=$1 + ; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; + + +CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ + SELECT jsonb_agg(content) FROM collections; +$$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; diff --git a/sql/002a_queryables.sql b/src/pgstac/sql/002a_queryables.sql similarity index 58% rename from sql/002a_queryables.sql rename to src/pgstac/sql/002a_queryables.sql index 78983bf0..304ff375 100644 --- a/sql/002a_queryables.sql +++ b/src/pgstac/sql/002a_queryables.sql @@ -11,15 +11,7 @@ CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); -INSERT INTO queryables (name, definition) VALUES -('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), -('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), -('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') -ON CONFLICT DO NOTHING; -INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES -('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') -ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( @@ -86,52 +78,204 @@ BEGIN END; $$ LANGUAGE PLPGSQL STABLE STRICT; -CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ + +DROP VIEW IF EXISTS pgstac_index; +CREATE VIEW pgstac_indexes AS +SELECT + i.schemaname, + i.tablename, + i.indexname, + indexdef, + COALESCE( + (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], + (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], + CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END + ) AS field, + pg_table_size(i.indexname::text) as index_size, + pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty +FROM + pg_indexes i +WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; + +DROP VIEW IF EXISTS pgstac_index_stats; +CREATE VIEW pgstac_indexes_stats AS +SELECT + i.schemaname, + i.tablename, + i.indexname, + indexdef, + COALESCE( + (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], + (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], + CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END + ) AS field, + pg_table_size(i.indexname::text) as index_size, + pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, + n_distinct, + most_common_vals::text::text[], + most_common_freqs::text::text[], + histogram_bounds::text::text[], + correlation +FROM + pg_indexes i + LEFT JOIN pg_stats s ON (s.tablename = i.indexname) +WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; + +set check_function_bodies to off; +CREATE OR REPLACE FUNCTION maintain_partition_queries( + part text DEFAULT 'items', + dropindexes boolean DEFAULT FALSE, + rebuildindexes boolean DEFAULT FALSE +) RETURNS SETOF text AS $$ DECLARE - queryable RECORD; + parent text; + level int; + isleaf bool; + collection collections%ROWTYPE; + subpart text; + baseidx text; + queryable_name text; + queryable_property_index_type text; + queryable_property_wrapper text; + queryable_parsed RECORD; + deletedidx pg_indexes%ROWTYPE; q text; + idx text; + collection_partition bigint; + _concurrently text := ''; BEGIN - FOR queryable IN + RAISE NOTICE 'Maintaining partition: %', part; + IF get_setting_bool('use_queue') THEN + _concurrently='CONCURRENTLY'; + END IF; + + -- Get root partition + SELECT parentrelid::text, pt.isleaf, pt.level + INTO parent, isleaf, level + FROM pg_partition_tree('items') pt + WHERE relid::text = part; + IF NOT FOUND THEN + RAISE NOTICE 'Partition % Does Not Exist In Partition Tree', part; + RETURN; + END IF; + + -- If this is a parent partition, recurse to leaves + IF NOT isleaf THEN + FOR subpart IN + SELECT relid::text + FROM pg_partition_tree(part) + WHERE relid::text != part + LOOP + RAISE NOTICE 'Recursing to %', subpart; + RETURN QUERY SELECT * FROM maintain_partition_queries(subpart, dropindexes, rebuildindexes); + END LOOP; + RETURN; -- Don't continue since not an end leaf + END IF; + + + -- Get collection + collection_partition := ((regexp_match(part, E'^_items_([0-9]+)'))[1])::bigint; + RAISE NOTICE 'COLLECTION PARTITION: %', collection_partition; + SELECT * INTO STRICT collection + FROM collections + WHERE key = collection_partition; + RAISE NOTICE 'COLLECTION ID: %s', collection.id; + + + -- Create temp table with existing indexes + CREATE TEMP TABLE existing_indexes ON COMMIT DROP AS + SELECT * + FROM pg_indexes + WHERE schemaname='pgstac' AND tablename=part; + + + -- Check if index exists for each queryable. + FOR + queryable_name, + queryable_property_index_type, + queryable_property_wrapper + IN SELECT - queryables.id as qid, - CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, - property_index_type, - expression - FROM - queryables - LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) - JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) + name, + COALESCE(property_index_type, 'BTREE'), + COALESCE(property_wrapper, 'to_text') + FROM queryables + WHERE + name NOT in ('id', 'datetime', 'geometry') + AND ( + collection_ids IS NULL + OR collection_ids = '{}'::text[] + OR collection.id = ANY (collection_ids) + ) + UNION ALL + SELECT 'datetime desc, end_datetime', 'BTREE', '' + UNION ALL + SELECT 'geometry', 'GIST', '' + UNION ALL + SELECT 'id', 'BTREE', '' + LOOP + baseidx := format( + $q$ ON %I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, + part, + queryable_property_index_type, + queryable_property_wrapper, + queryable_name + ); + RAISE NOTICE 'BASEIDX: %', baseidx; + RAISE NOTICE 'IDXSEARCH: %', format($q$[(']%s[')]$q$, queryable_name); + -- If index already exists, delete it from existing indexes type table + FOR deletedidx IN + DELETE FROM existing_indexes + WHERE indexdef ~* format($q$[(']%s[')]$q$, queryable_name) + RETURNING * LOOP - q := format( - $q$ - CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); - $q$, - format('%s_%s_idx', queryable.part, queryable.qid), - queryable.part, - COALESCE(queryable.property_index_type, 'to_text'), - queryable.expression - ); - RAISE NOTICE '%',q; - EXECUTE q; + RAISE NOTICE 'EXISTING INDEX: %', deletedidx; + IF NOT FOUND THEN -- index did not exist, create it + RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); + ELSIF rebuildindexes THEN + RETURN NEXT format('REINDEX %I %s;', deletedidx.indexname, _concurrently); + END IF; + END LOOP; + END LOOP; + + -- Remove indexes that were not expected + FOR idx IN SELECT indexname::text FROM existing_indexes + LOOP + RAISE WARNING 'Index: % is not defined by queryables.', idx; + IF dropindexes THEN + RETURN NEXT format('DROP INDEX IF EXISTS %I;', idx); + END IF; END LOOP; + + DROP TABLE existing_indexes; + RAISE NOTICE 'Returning from maintain_partition_queries.'; RETURN; + END; $$ LANGUAGE PLPGSQL; +CREATE OR REPLACE FUNCTION maintain_partitions( + part text DEFAULT 'items', + dropindexes boolean DEFAULT FALSE, + rebuildindexes boolean DEFAULT FALSE +) RETURNS VOID AS $$ + WITH t AS ( + SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q + ) SELECT count(*) FROM t; +$$ LANGUAGE SQL; + + CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN -PERFORM create_queryable_indexes(); -RETURN NEW; + PERFORM maintain_partitions(); + RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); -CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections -FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); - CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE diff --git a/sql/002b_cql.sql b/src/pgstac/sql/002b_cql.sql similarity index 100% rename from sql/002b_cql.sql rename to src/pgstac/sql/002b_cql.sql diff --git a/sql/003_items.sql b/src/pgstac/sql/003a_items.sql similarity index 82% rename from sql/003_items.sql rename to src/pgstac/sql/003a_items.sql index 9d29128d..a10f556b 100644 --- a/sql/003_items.sql +++ b/src/pgstac/sql/003a_items.sql @@ -14,9 +14,43 @@ CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; - ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; +CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ +DECLARE + p text; + t timestamptz := clock_timestamp(); +BEGIN + RAISE NOTICE 'Updating partition stats %', t; + FOR p IN SELECT DISTINCT partition + FROM newdata n JOIN partition_sys_meta p + ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) + LOOP + PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); + END LOOP; + RAISE NOTICE 't: % %', t, clock_timestamp() - t; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + +CREATE OR REPLACE TRIGGER items_after_insert_trigger +AFTER INSERT ON items +REFERENCING NEW TABLE AS newdata +FOR EACH STATEMENT +EXECUTE FUNCTION partition_after_triggerfunc(); + +CREATE OR REPLACE TRIGGER items_after_update_trigger +AFTER DELETE ON items +REFERENCING OLD TABLE AS newdata +FOR EACH STATEMENT +EXECUTE FUNCTION partition_after_triggerfunc(); + +CREATE OR REPLACE TRIGGER items_after_delete_trigger +AFTER UPDATE ON items +REFERENCING NEW TABLE AS newdata +FOR EACH STATEMENT +EXECUTE FUNCTION partition_after_triggerfunc(); + CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; @@ -155,36 +189,28 @@ CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; + part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; - WITH ranges AS ( + + FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, - stac_daterange(n.content->'properties') as dtr - FROM newdata n + stac_daterange(n.content->'properties') as dtr, + partition_trunc + FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, - lower(dtr) as datetime, - upper(dtr) as end_datetime, - (partition_name( - collection, - lower(dtr) - )).partition_name as name - FROM ranges - ) - INSERT INTO partitions (collection, datetime_range, end_datetime_range) - SELECT - collection, - tstzrange(min(datetime), max(datetime), '[]') as datetime_range, - tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range - FROM p - GROUP BY collection, name - ON CONFLICT (name) DO UPDATE SET - datetime_range = EXCLUDED.datetime_range, - end_datetime_range = EXCLUDED.end_datetime_range - ; + COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, + tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, + tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange + FROM t + GROUP BY 1,2 + ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP + RAISE NOTICE 'Partition %', part; + END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN @@ -192,6 +218,7 @@ BEGIN SELECT (content_dehydrate(content)).* FROM newdata; + RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items @@ -199,6 +226,7 @@ BEGIN (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; + RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( @@ -215,6 +243,7 @@ BEGIN SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; + RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; @@ -235,8 +264,6 @@ CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); - - CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE @@ -245,11 +272,11 @@ BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; -$$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; +$$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); -$$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; +$$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE @@ -317,49 +344,3 @@ UPDATE collections SET ) ; $$ LANGUAGE SQL; - - -CREATE OR REPLACE PROCEDURE analyze_items() AS $$ -DECLARE -q text; -BEGIN -FOR q IN - SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) - FROM pg_stat_user_tables - WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) -LOOP - RAISE NOTICE '%', q; - EXECUTE q; - COMMIT; -END LOOP; -END; -$$ LANGUAGE PLPGSQL; - - -CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ -DECLARE - q text; -BEGIN - FOR q IN - SELECT - FORMAT( - 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', - nsp.nspname, - cls.relname, - con.conname - ) - - FROM pg_constraint AS con - JOIN pg_class AS cls - ON con.conrelid = cls.oid - JOIN pg_namespace AS nsp - ON cls.relnamespace = nsp.oid - WHERE convalidated = FALSE AND contype in ('c','f') - AND nsp.nspname = 'pgstac' - LOOP - RAISE NOTICE '%', q; - EXECUTE q; - COMMIT; - END LOOP; -END; -$$ LANGUAGE PLPGSQL; diff --git a/src/pgstac/sql/003b_partitions.sql b/src/pgstac/sql/003b_partitions.sql new file mode 100644 index 00000000..b12ba18c --- /dev/null +++ b/src/pgstac/sql/003b_partitions.sql @@ -0,0 +1,456 @@ +CREATE TABLE partition_stats ( + partition text PRIMARY KEY, + dtrange tstzrange, + edtrange tstzrange, + spatial geometry, + last_updated timestamptz, + keys text[] +) WITH (FILLFACTOR=90); + +CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); + + +CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ + WITH t AS ( + SELECT regexp_matches( + expr, + E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' + ) AS m + ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t + ; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; + +CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ +DECLARE + expr text := pg_get_constraintdef(coid); + matches timestamptz[]; +BEGIN + IF expr LIKE '%NULL%' THEN + dt := tstzrange(null::timestamptz, null::timestamptz); + edt := tstzrange(null::timestamptz, null::timestamptz); + RETURN; + END IF; + WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) + SELECT array_agg(f::timestamptz) INTO matches FROM f; + IF cardinality(matches) = 4 THEN + dt := tstzrange(matches[1], matches[2],'[]'); + edt := tstzrange(matches[3], matches[4], '[]'); + RETURN; + ELSIF cardinality(matches) = 2 THEN + edt := tstzrange(matches[1], matches[2],'[]'); + RETURN; + END IF; + RETURN; +END; +$$ LANGUAGE PLPGSQL STABLE STRICT; + +CREATE OR REPLACE VIEW partition_sys_meta AS +SELECT + relid::text as partition, + replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) + ELSE pg_get_expr(parent.relpartbound, parent.oid) + END, 'FOR VALUES IN (''',''), ''')','') AS collection, + level, + c.reltuples, + c.relhastriggers, + COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, + COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, + COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange +FROM + pg_partition_tree('items') + JOIN pg_class c ON (relid::regclass = c.oid) + JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) + LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') +WHERE isleaf +; + +CREATE VIEW partitions AS +SELECT * FROM partition_sys_meta LEFT JOIN partition_stats USING (partition); + +CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ +DECLARE +BEGIN + PERFORM run_or_queue( + format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) + ); +END; +$$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ +DECLARE + dtrange tstzrange; + edtrange tstzrange; + cdtrange tstzrange; + cedtrange tstzrange; + extent geometry; + collection text; +BEGIN + RAISE NOTICE 'Updating stats for %.', _partition; + EXECUTE format( + $q$ + SELECT + tstzrange(min(datetime), max(datetime),'[]'), + tstzrange(min(end_datetime), max(end_datetime), '[]') + FROM %I + $q$, + _partition + ) INTO dtrange, edtrange; + extent := st_estimatedextent('pgstac', _partition, 'geometry'); + INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) + SELECT _partition, dtrange, edtrange, extent, now() + ON CONFLICT (partition) DO + UPDATE SET + dtrange=EXCLUDED.dtrange, + edtrange=EXCLUDED.edtrange, + spatial=EXCLUDED.spatial, + last_updated=EXCLUDED.last_updated + ; + SELECT + constraint_dtrange, constraint_edtrange, partitions.collection + INTO cdtrange, cedtrange, collection + FROM partitions WHERE partition = _partition; + + RAISE NOTICE 'Checking if we need to modify constraints.'; + IF + (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) + AND NOT istrigger + THEN + RAISE NOTICE 'Modifying Constraints'; + RAISE NOTICE 'Existing % %', cdtrange, cedtrange; + RAISE NOTICE 'New % %', dtrange, edtrange; + PERFORM drop_table_constraints(_partition); + PERFORM create_table_constraints(_partition, dtrange, edtrange); + END IF; + RAISE NOTICE 'Checking if we need to update collection extents.'; + IF get_setting_bool('update_collection_extent') THEN + RAISE NOTICE 'updating collection extent for %', collection; + PERFORM run_or_queue(format($q$ + UPDATE collections + SET content = jsonb_set_lax( + content, + '{extent}'::text[], + collection_extent(%L), + true, + 'use_json_null' + ) WHERE id=%L + ; + $q$, collection, collection)); + ELSE + RAISE NOTICE 'Not updating collection extent for %', collection; + END IF; +END; +$$ LANGUAGE PLPGSQL STRICT; + + +CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ +DECLARE + c RECORD; + parent_name text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + parent_name := format('_items_%s', c.key); + + + IF c.partition_trunc = 'year' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); + ELSE + partition_name := parent_name; + partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); + END IF; + IF partition_range IS NULL THEN + partition_range := tstzrange( + date_trunc(c.partition_trunc::text, dt), + date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval + ); + END IF; + RETURN; + +END; +$$ LANGUAGE PLPGSQL STABLE; + + +CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ +DECLARE + q text; +BEGIN + IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN + RETURN NULL; + END IF; + FOR q IN SELECT FORMAT( + $q$ + ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; + $q$, + t, + conname + ) FROM pg_constraint + WHERE conrelid=t::regclass::oid AND contype='c' + LOOP + EXECUTE q; + END LOOP; + RETURN t; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ +DECLARE + q text; +BEGIN + IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN + RETURN NULL; + END IF; + RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; + IF _dtrange = 'empty' AND _edtrange = 'empty' THEN + q :=format( + $q$ + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + $q$, + t, + format('%s_dt', t), + t, + format('%s_dt', t) + ); + ELSE + q :=format( + $q$ + ALTER TABLE %I + ADD CONSTRAINT %I + CHECK ( + (datetime >= %L) + AND (datetime <= %L) + AND (end_datetime >= %L) + AND (end_datetime <= %L) + ) NOT VALID + ; + ALTER TABLE %I + VALIDATE CONSTRAINT %I + ; + $q$, + t, + format('%s_dt', t), + lower(_dtrange), + upper(_dtrange), + lower(_edtrange), + upper(_edtrange), + t, + format('%s_dt', t) + ); + END IF; + PERFORM run_or_queue(q); + RETURN t; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION check_partition( + _collection text, + _dtrange tstzrange, + _edtrange tstzrange +) RETURNS text AS $$ +DECLARE + c RECORD; + pm RECORD; + _partition_name text; + _partition_dtrange tstzrange; + _constraint_dtrange tstzrange; + _constraint_edtrange tstzrange; + q text; + deferrable_q text; + err_context text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + + IF c.partition_trunc IS NOT NULL THEN + _partition_dtrange := tstzrange( + date_trunc(c.partition_trunc, lower(_dtrange)), + date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, + '[)' + ); + ELSE + _partition_dtrange := '[-infinity, infinity]'::tstzrange; + END IF; + + IF NOT _partition_dtrange @> _dtrange THEN + RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; + END IF; + + + IF c.partition_trunc = 'year' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); + ELSIF c.partition_trunc = 'month' THEN + _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); + ELSE + _partition_name := format('_items_%s', c.key); + END IF; + + SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; + IF FOUND THEN + RAISE NOTICE '% % %', _edtrange, _dtrange, pm; + _constraint_edtrange := + tstzrange( + least( + lower(_edtrange), + nullif(lower(pm.constraint_edtrange), '-infinity') + ), + greatest( + upper(_edtrange), + nullif(upper(pm.constraint_edtrange), 'infinity') + ), + '[]' + ); + _constraint_dtrange := + tstzrange( + least( + lower(_dtrange), + nullif(lower(pm.constraint_dtrange), '-infinity') + ), + greatest( + upper(_dtrange), + nullif(upper(pm.constraint_dtrange), 'infinity') + ), + '[]' + ); + + IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN + RETURN pm.partition; + ELSE + PERFORM drop_table_constraints(_partition_name); + END IF; + ELSE + _constraint_edtrange := _edtrange; + _constraint_dtrange := _dtrange; + END IF; + RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; + IF c.partition_trunc IS NULL THEN + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + _partition_name, + _collection, + concat(_partition_name,'_pk'), + _partition_name + ); + ELSE + q := format( + $q$ + CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); + CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); + CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); + $q$, + format('_items_%s', c.key), + _collection, + _partition_name, + format('_items_%s', c.key), + lower(_partition_dtrange), + upper(_partition_dtrange), + format('%s_pk', _partition_name), + _partition_name + ); + END IF; + + BEGIN + EXECUTE q; + EXCEPTION + WHEN duplicate_table THEN + RAISE NOTICE 'Partition % already exists.', _partition_name; + WHEN others THEN + GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; + RAISE INFO 'Error Name:%',SQLERRM; + RAISE INFO 'Error State:%', SQLSTATE; + RAISE INFO 'Error Context:%', err_context; + END; + PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); + PERFORM maintain_partitions(_partition_name); + PERFORM update_partition_stats_q(_partition_name, true); + RETURN _partition_name; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + + +CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ +DECLARE + c RECORD; + q text; + from_trunc text; +BEGIN + SELECT * INTO c FROM pgstac.collections WHERE id=_collection; + IF NOT FOUND THEN + RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; + END IF; + IF triggered THEN + RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; + ELSE + RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; + IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN + RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; + RETURN _collection; + END IF; + END IF; + + IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN + EXECUTE format( + $q$ + CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; + DROP TABLE IF EXISTS %I CASCADE; + WITH p AS ( + SELECT + collection, + CASE WHEN %L IS NULL THEN '-infinity'::timestamptz + ELSE date_trunc(%L, datetime) + END as d, + tstzrange(min(datetime),max(datetime),'[]') as dtrange, + tstzrange(min(datetime),max(datetime),'[]') as edtrange + FROM changepartitionstaging + GROUP BY 1,2 + ) SELECT check_partition(collection, dtrange, edtrange) FROM p; + INSERT INTO items SELECT * FROM changepartitionstaging; + DROP TABLE changepartitionstaging; + $q$, + concat('_items_', c.key), + concat('_items_', c.key), + c.partition_trunc, + c.partition_trunc + ); + END IF; + RETURN _collection; +END; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ +DECLARE + q text; + partition_name text := format('_items_%s', NEW.key); + partition_exists boolean := false; + partition_empty boolean := true; + err_context text; + loadtemp boolean := FALSE; +BEGIN + RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; + IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN + PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); + END IF; + RETURN NEW; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE TRIGGER collections_trigger AFTER +INSERT +OR +UPDATE ON collections +FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); diff --git a/sql/004_search.sql b/src/pgstac/sql/004_search.sql similarity index 98% rename from sql/004_search.sql rename to src/pgstac/sql/004_search.sql index 108f409c..3db41fde 100644 --- a/sql/004_search.sql +++ b/src/pgstac/sql/004_search.sql @@ -1,10 +1,10 @@ -CREATE VIEW partition_steps AS +CREATE OR REPLACE VIEW partition_steps AS SELECT - name, - date_trunc('month',lower(datetime_range)) as sdate, - date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate - FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange - ORDER BY datetime_range ASC + partition as name, + date_trunc('month',lower(partition_dtrange)) as sdate, + date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate + FROM partitions WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange + ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION chunker( @@ -424,6 +424,7 @@ CREATE TABLE IF NOT EXISTS searches( usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); + CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, @@ -549,11 +550,9 @@ BEGIN ; RETURN sw; END; -$$ LANGUAGE PLPGSQL ; - +$$ LANGUAGE PLPGSQL SECURITY DEFINER; -DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, @@ -596,7 +595,7 @@ BEGIN RETURN search; END; -$$ LANGUAGE PLPGSQL; +$$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE @@ -778,7 +777,7 @@ collection := jsonb_build_object( RETURN collection; END; -$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; +$$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ diff --git a/sql/005_tileutils.sql b/src/pgstac/sql/005_tileutils.sql similarity index 100% rename from sql/005_tileutils.sql rename to src/pgstac/sql/005_tileutils.sql diff --git a/sql/006_tilesearch.sql b/src/pgstac/sql/006_tilesearch.sql similarity index 100% rename from sql/006_tilesearch.sql rename to src/pgstac/sql/006_tilesearch.sql diff --git a/src/pgstac/sql/997_maintenance.sql b/src/pgstac/sql/997_maintenance.sql new file mode 100644 index 00000000..d0d418a7 --- /dev/null +++ b/src/pgstac/sql/997_maintenance.sql @@ -0,0 +1,93 @@ + +DROP FUNCTION IF EXISTS analyze_items; +CREATE OR REPLACE PROCEDURE analyze_items() AS $$ +DECLARE + q text; + timeout_ts timestamptz; +BEGIN + timeout_ts := statement_timestamp() + queue_timeout(); + WHILE clock_timestamp() < timeout_ts LOOP + RAISE NOTICE '% % %', clock_timestamp(), timeout_ts, current_setting('statement_timeout', TRUE); + SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q + FROM pg_stat_user_tables + WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; + IF NOT FOUND THEN + EXIT; + END IF; + RAISE NOTICE '%', q; + EXECUTE q; + COMMIT; + RAISE NOTICE '%', queue_timeout(); + END LOOP; +END; +$$ LANGUAGE PLPGSQL; + + +DROP FUNCTION IF EXISTS validate_constraints; +CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ +DECLARE + q text; +BEGIN + FOR q IN + SELECT + FORMAT( + 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', + nsp.nspname, + cls.relname, + con.conname + ) + + FROM pg_constraint AS con + JOIN pg_class AS cls + ON con.conrelid = cls.oid + JOIN pg_namespace AS nsp + ON cls.relnamespace = nsp.oid + WHERE convalidated = FALSE AND contype in ('c','f') + AND nsp.nspname = 'pgstac' + LOOP + RAISE NOTICE '%', q; + PERFORM run_or_queue(q); + COMMIT; + END LOOP; +END; +$$ LANGUAGE PLPGSQL; + + +CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ +DECLARE + geom_extent geometry; + mind timestamptz; + maxd timestamptz; + extent jsonb; +BEGIN + IF runupdate THEN + PERFORM update_partition_stats_q(partition) + FROM partitions WHERE collection=_collection; + END IF; + SELECT + min(lower(dtrange)), + max(upper(edtrange)), + st_extent(spatial) + INTO + mind, + maxd, + geom_extent + FROM partitions + WHERE collection=_collection; + + IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN + extent := jsonb_build_object( + 'extent', jsonb_build_object( + 'spatial', jsonb_build_object( + 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) + ), + 'temporal', jsonb_build_object( + 'interval', to_jsonb(array[array[mind, maxd]]) + ) + ) + ); + RETURN extent; + END IF; + RETURN NULL; +END; +$$ LANGUAGE PLPGSQL; diff --git a/src/pgstac/sql/998_idempotent_post.sql b/src/pgstac/sql/998_idempotent_post.sql new file mode 100644 index 00000000..a8bf8175 --- /dev/null +++ b/src/pgstac/sql/998_idempotent_post.sql @@ -0,0 +1,40 @@ +INSERT INTO queryables (name, definition) VALUES +('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), +('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), +('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') +ON CONFLICT DO NOTHING; + +INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES +('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') +ON CONFLICT DO NOTHING; + + +INSERT INTO pgstac_settings (name, value) VALUES + ('context', 'off'), + ('context_estimated_count', '100000'), + ('context_estimated_cost', '100000'), + ('context_stats_ttl', '1 day'), + ('default_filter_lang', 'cql2-json'), + ('additional_properties', 'true'), + ('use_queue', 'false'), + ('queue_timeout', '10 minutes'), + ('update_collection_extent', 'true') +ON CONFLICT DO NOTHING +; + + +GRANT USAGE ON SCHEMA pgstac to pgstac_read; +GRANT ALL ON SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON SCHEMA pgstac to pgstac_admin; + +-- pgstac_read role limited to using function apis +GRANT EXECUTE ON FUNCTION search TO pgstac_read; +GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; +GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; +GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; + +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; +GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; + +SELECT update_partition_stats_q(partition) FROM partitions; diff --git a/src/pgstac/sql/999_version.sql b/src/pgstac/sql/999_version.sql new file mode 100644 index 00000000..611a74d8 --- /dev/null +++ b/src/pgstac/sql/999_version.sql @@ -0,0 +1 @@ +SELECT set_version('0.7.0'); diff --git a/test/basic/cql2_searches.sql b/src/pgstac/tests/basic/cql2_searches.sql similarity index 100% rename from test/basic/cql2_searches.sql rename to src/pgstac/tests/basic/cql2_searches.sql diff --git a/test/basic/cql2_searches.sql.out b/src/pgstac/tests/basic/cql2_searches.sql.out similarity index 99% rename from test/basic/cql2_searches.sql.out rename to src/pgstac/tests/basic/cql2_searches.sql.out index 02a04646..cda464f5 100644 --- a/test/basic/cql2_searches.sql.out +++ b/src/pgstac/tests/basic/cql2_searches.sql.out @@ -157,8 +157,3 @@ SELECT jsonb_path_query(search('{"token":"prev:pgstac-test-item-0054","sortby":[ "pgstac-test-item-0085" "pgstac-test-item-0012" "pgstac-test-item-0048" - - - -\set QUIET 1 -\set ECHO none diff --git a/test/basic/cql_searches.sql b/src/pgstac/tests/basic/cql_searches.sql similarity index 100% rename from test/basic/cql_searches.sql rename to src/pgstac/tests/basic/cql_searches.sql diff --git a/test/basic/cql_searches.sql.out b/src/pgstac/tests/basic/cql_searches.sql.out similarity index 99% rename from test/basic/cql_searches.sql.out rename to src/pgstac/tests/basic/cql_searches.sql.out index 75e6c033..40a3a285 100644 --- a/test/basic/cql_searches.sql.out +++ b/src/pgstac/tests/basic/cql_searches.sql.out @@ -58,6 +58,3 @@ SELECT hash from search_query('{"collections":["pgstac-test-collection"]}'); SELECT search from search_query('{"collections":["pgstac-test-collection"]}'); {"collections": ["pgstac-test-collection"]} - -\set QUIET 1 -\set ECHO none diff --git a/src/pgstac/tests/basic/partitions.sql b/src/pgstac/tests/basic/partitions.sql new file mode 100644 index 00000000..5dac08ce --- /dev/null +++ b/src/pgstac/tests/basic/partitions.sql @@ -0,0 +1,76 @@ +SET pgstac.use_queue=FALSE; +SELECT get_setting_bool('use_queue'); +SET pgstac.update_collection_extent=TRUE; +SELECT get_setting_bool('update_collection_extent'); +--create base data to use with tests +CREATE TEMP TABLE test_items AS +SELECT jsonb_build_object( + 'id', concat('pgstactest-partitioned-', (row_number() over ())::text), + 'collection', 'pgstactest-partitioned', + 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, + 'properties', jsonb_build_object( 'datetime', g::text) +) as content FROM generate_series('2020-01-01'::timestamptz, '2022-01-01'::timestamptz, '1 week'::interval) g; + +--test non-partitioned collection +INSERT INTO collections (content) VALUES ('{"id":"pgstactest-partitioned"}'); +INSERT INTO items_staging(content) +SELECT content FROM test_items; +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned'; + +--test collection partioned by year +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-year"}', 'year'); +INSERT INTO items_staging(content) +SELECT content || '{"collection":"pgstactest-partitioned-year"}'::jsonb FROM test_items; +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-year'; + +--test collection partioned by month +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-month"}', 'month'); +INSERT INTO items_staging(content) +SELECT content || '{"collection":"pgstactest-partitioned-month"}'::jsonb FROM test_items; +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-month'; + +--test repartitioning from year to non partitioned +UPDATE collections SET partition_trunc=NULL WHERE id='pgstactest-partitioned-year'; +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-year'; +SELECT count(*) FROM items WHERE collection='pgstactest-partitioned-year'; + +--test repartitioning from non-partitioned to year +UPDATE collections SET partition_trunc='year' WHERE id='pgstactest-partitioned'; +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned'; +SELECT count(*) FROM items WHERE collection='pgstactest-partitioned'; + +--check that partition stats have been updated +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned' and spatial IS NULL; + +--test noop for repartitioning +UPDATE collections SET content=content || '{"foo":"bar"}'::jsonb WHERE id='pgstactest-partitioned-month'; +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-month'; +SELECT count(*) FROM items WHERE collection='pgstactest-partitioned-month'; + +--test using query queue +SET pgstac.use_queue=TRUE; +SELECT get_setting_bool('use_queue'); + +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-q"}', 'month'); +INSERT INTO items_staging(content) +SELECT content || '{"collection":"pgstactest-partitioned-q"}'::jsonb FROM test_items; +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q'; + +--check that partition stats haven't been updated +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q' and spatial IS NULL; + +--check that queue has items +SELECT count(*)>0 FROM query_queue; + +--run queue items to update partition stats +SELECT run_queued_queries_intransaction()>0; + +--check that queue has been emptied +SELECT count(*) FROM query_queue; +SELECT run_queued_queries_intransaction(); + +--check that partition stats have been updated +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q' and spatial IS NULL; + +--check that collection extents have been updated +SELECT DISTINCT content->'extent' FROM collections WHERE id LIKE 'pgstactest-partitioned%'; diff --git a/src/pgstac/tests/basic/partitions.sql.out b/src/pgstac/tests/basic/partitions.sql.out new file mode 100644 index 00000000..b6386fc3 --- /dev/null +++ b/src/pgstac/tests/basic/partitions.sql.out @@ -0,0 +1,114 @@ +SET pgstac.use_queue=FALSE; +SET +SELECT get_setting_bool('use_queue'); +f +SET pgstac.update_collection_extent=TRUE; +SET +SELECT get_setting_bool('update_collection_extent'); +t +--create base data to use with tests +CREATE TEMP TABLE test_items AS +SELECT jsonb_build_object( +'id', concat('pgstactest-partitioned-', (row_number() over ())::text), +'collection', 'pgstactest-partitioned', +'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, +'properties', jsonb_build_object( 'datetime', g::text) +) as content FROM generate_series('2020-01-01'::timestamptz, '2022-01-01'::timestamptz, '1 week'::interval) g; +SELECT 105 +--test non-partitioned collection +INSERT INTO collections (content) VALUES ('{"id":"pgstactest-partitioned"}'); +INSERT 0 1 +INSERT INTO items_staging(content) +SELECT content FROM test_items; +INSERT 0 105 +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned'; +1 + +--test collection partioned by year +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-year"}', 'year'); +INSERT 0 1 +INSERT INTO items_staging(content) +SELECT content || '{"collection":"pgstactest-partitioned-year"}'::jsonb FROM test_items; +INSERT 0 105 +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-year'; +2 + +--test collection partioned by month +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-month"}', 'month'); +INSERT 0 1 +INSERT INTO items_staging(content) +SELECT content || '{"collection":"pgstactest-partitioned-month"}'::jsonb FROM test_items; +INSERT 0 105 +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-month'; +24 + +--test repartitioning from year to non partitioned +UPDATE collections SET partition_trunc=NULL WHERE id='pgstactest-partitioned-year'; +UPDATE 1 +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-year'; +1 + +SELECT count(*) FROM items WHERE collection='pgstactest-partitioned-year'; +105 + +--test repartitioning from non-partitioned to year +UPDATE collections SET partition_trunc='year' WHERE id='pgstactest-partitioned'; +UPDATE 1 +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned'; +2 + +SELECT count(*) FROM items WHERE collection='pgstactest-partitioned'; +105 + +--check that partition stats have been updated +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned' and spatial IS NULL; +0 + +--test noop for repartitioning +UPDATE collections SET content=content || '{"foo":"bar"}'::jsonb WHERE id='pgstactest-partitioned-month'; +UPDATE 1 +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-month'; +24 + +SELECT count(*) FROM items WHERE collection='pgstactest-partitioned-month'; +105 + +--test using query queue +SET pgstac.use_queue=TRUE; +SET +SELECT get_setting_bool('use_queue'); +t +INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-q"}', 'month'); +INSERT 0 1 +INSERT INTO items_staging(content) +SELECT content || '{"collection":"pgstactest-partitioned-q"}'::jsonb FROM test_items; +INSERT 0 105 +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q'; +24 + +--check that partition stats haven't been updated +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q' and spatial IS NULL; +24 + +--check that queue has items +SELECT count(*)>0 FROM query_queue; +t + +--run queue items to update partition stats +SELECT run_queued_queries_intransaction()>0; +t + +--check that queue has been emptied +SELECT count(*) FROM query_queue; +0 + +SELECT run_queued_queries_intransaction(); +0 + +--check that partition stats have been updated +SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q' and spatial IS NULL; +0 + +--check that collection extents have been updated +SELECT DISTINCT content->'extent' FROM collections WHERE id LIKE 'pgstactest-partitioned%'; +{"extent": {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}}} diff --git a/test/basic/xyz_searches.sql b/src/pgstac/tests/basic/xyz_searches.sql similarity index 100% rename from test/basic/xyz_searches.sql rename to src/pgstac/tests/basic/xyz_searches.sql diff --git a/test/basic/xyz_searches.sql.out b/src/pgstac/tests/basic/xyz_searches.sql.out similarity index 98% rename from test/basic/xyz_searches.sql.out rename to src/pgstac/tests/basic/xyz_searches.sql.out index 41c33619..be5c1f43 100644 --- a/test/basic/xyz_searches.sql.out +++ b/src/pgstac/tests/basic/xyz_searches.sql.out @@ -17,6 +17,3 @@ SELECT xyzsearch(16792, 26892, 16, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"includ SELECT xyzsearch(16792, 26892, 16, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, exitwhenfull => false, skipcovered => false); {"type": "FeatureCollection", "features": [{"id": "pgstac-test-item-0098", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0091", "collection": "pgstac-test-collection"}]} - -\set QUIET 1 -\set ECHO none diff --git a/test/pgtap.sql b/src/pgstac/tests/pgtap.sql similarity index 57% rename from test/pgtap.sql rename to src/pgstac/tests/pgtap.sql index a75d261d..47719b10 100644 --- a/test/pgtap.sql +++ b/src/pgstac/tests/pgtap.sql @@ -9,7 +9,6 @@ \timing off -- Revert all changes on failure. -\set ON_ERROR_ROLLBACK 1 \set ON_ERROR_STOP true -- Load the TAP functions. @@ -19,23 +18,23 @@ SET SEARCH_PATH TO pgstac, pgtap, public; SET CLIENT_MIN_MESSAGES TO 'warning'; -- Plan the tests. -SELECT plan(72); +SELECT plan(71); --SELECT * FROM no_plan(); -- Run the tests. -- Core -\i test//pgtap/001_core.sql -\i test/pgtap/001a_jsonutils.sql -\i test/pgtap/001b_cursorutils.sql -\i test/pgtap/001s_stacutils.sql -\i test/pgtap/002_collections.sql -\i test/pgtap/002a_queryables.sql -\i test/pgtap/003_items.sql -\i test/pgtap/004_search.sql -\i test/pgtap/005_tileutils.sql -\i test/pgtap/006_tilesearch.sql -\i test/pgtap/999_version.sql +\i tests/pgtap/001_core.sql +\i tests/pgtap/001a_jsonutils.sql +\i tests/pgtap/001b_cursorutils.sql +\i tests/pgtap/001s_stacutils.sql +\i tests/pgtap/002_collections.sql +\i tests/pgtap/002a_queryables.sql +\i tests/pgtap/003_items.sql +\i tests/pgtap/004_search.sql +\i tests/pgtap/005_tileutils.sql +\i tests/pgtap/006_tilesearch.sql +\i tests/pgtap/999_version.sql -- Finish the tests and clean up. SELECT * FROM finish(); diff --git a/test/pgtap/001_core.sql b/src/pgstac/tests/pgtap/001_core.sql similarity index 100% rename from test/pgtap/001_core.sql rename to src/pgstac/tests/pgtap/001_core.sql diff --git a/test/pgtap/001a_jsonutils.sql b/src/pgstac/tests/pgtap/001a_jsonutils.sql similarity index 100% rename from test/pgtap/001a_jsonutils.sql rename to src/pgstac/tests/pgtap/001a_jsonutils.sql diff --git a/test/pgtap/001b_cursorutils.sql b/src/pgstac/tests/pgtap/001b_cursorutils.sql similarity index 100% rename from test/pgtap/001b_cursorutils.sql rename to src/pgstac/tests/pgtap/001b_cursorutils.sql diff --git a/test/pgtap/001s_stacutils.sql b/src/pgstac/tests/pgtap/001s_stacutils.sql similarity index 100% rename from test/pgtap/001s_stacutils.sql rename to src/pgstac/tests/pgtap/001s_stacutils.sql diff --git a/test/pgtap/002_collections.sql b/src/pgstac/tests/pgtap/002_collections.sql similarity index 100% rename from test/pgtap/002_collections.sql rename to src/pgstac/tests/pgtap/002_collections.sql diff --git a/test/pgtap/002a_queryables.sql b/src/pgstac/tests/pgtap/002a_queryables.sql similarity index 84% rename from test/pgtap/002a_queryables.sql rename to src/pgstac/tests/pgtap/002a_queryables.sql index d297c00e..a68820ce 100644 --- a/test/pgtap/002a_queryables.sql +++ b/src/pgstac/tests/pgtap/002a_queryables.sql @@ -1,10 +1,3 @@ -SELECT results_eq( - $$ SELECT property_wrapper FROM queryables WHERE name='eo:cloud_cover'; $$, - $$ SELECT 'to_int'; $$, - 'Make sure that cloud_cover is set to to_int wrapper.' -); - - SELECT results_eq( $$ SELECT sort_sqlorderby('{"sortby":{"field":"properties.eo:cloud_cover"}}'); $$, $$ SELECT sort_sqlorderby('{"sortby":{"field":"eo:cloud_cover"}}'); $$, @@ -28,7 +21,7 @@ SELECT results_eq( ); DELETE FROM collections WHERE id = 'pgstac-test-collection'; -\copy collections (content) FROM 'test/testdata/collections.ndjson'; +\copy collections (content) FROM 'tests/testdata/collections.ndjson'; SELECT results_eq( $$ SELECT get_queryables('pgstac-test-collection') -> 'properties' ? 'datetime'; $$, diff --git a/test/pgtap/003_items.sql b/src/pgstac/tests/pgtap/003_items.sql similarity index 98% rename from test/pgtap/003_items.sql rename to src/pgstac/tests/pgtap/003_items.sql index 67dcf124..6e785e94 100644 --- a/test/pgtap/003_items.sql +++ b/src/pgstac/tests/pgtap/003_items.sql @@ -21,7 +21,7 @@ SELECT has_function('pgstac'::name, 'collection_temporal_extent', ARRAY['text']) SELECT has_function('pgstac'::name, 'update_collection_extents', '{}'::text[]); DELETE FROM collections WHERE id = 'pgstac-test-collection'; -\copy collections (content) FROM 'test/testdata/collections.ndjson'; +\copy collections (content) FROM 'tests/testdata/collections.ndjson'; SELECT create_item('{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}'); diff --git a/test/pgtap/004_search.sql b/src/pgstac/tests/pgtap/004_search.sql similarity index 99% rename from test/pgtap/004_search.sql rename to src/pgstac/tests/pgtap/004_search.sql index 1e8fcc96..17d23ff6 100644 --- a/test/pgtap/004_search.sql +++ b/src/pgstac/tests/pgtap/004_search.sql @@ -1,6 +1,6 @@ -- CREATE fixtures for testing search - as tests are run within a transaction, these will not persist -\copy items_staging (content) FROM 'test/testdata/items.ndjson' +\copy items_staging (content) FROM 'tests/testdata/items.ndjson' SET pgstac.context TO 'on'; SET pgstac."default_filter_lang" TO 'cql-json'; diff --git a/test/pgtap/005_tileutils.sql b/src/pgstac/tests/pgtap/005_tileutils.sql similarity index 100% rename from test/pgtap/005_tileutils.sql rename to src/pgstac/tests/pgtap/005_tileutils.sql diff --git a/test/pgtap/006_tilesearch.sql b/src/pgstac/tests/pgtap/006_tilesearch.sql similarity index 100% rename from test/pgtap/006_tilesearch.sql rename to src/pgstac/tests/pgtap/006_tilesearch.sql diff --git a/test/pgtap/999_version.sql b/src/pgstac/tests/pgtap/999_version.sql similarity index 100% rename from test/pgtap/999_version.sql rename to src/pgstac/tests/pgtap/999_version.sql diff --git a/test/testdata/collections.json b/src/pgstac/tests/testdata/collections.json similarity index 100% rename from test/testdata/collections.json rename to src/pgstac/tests/testdata/collections.json diff --git a/test/testdata/collections.ndjson b/src/pgstac/tests/testdata/collections.ndjson similarity index 100% rename from test/testdata/collections.ndjson rename to src/pgstac/tests/testdata/collections.ndjson diff --git a/test/testdata/items.ndjson b/src/pgstac/tests/testdata/items.ndjson similarity index 100% rename from test/testdata/items.ndjson rename to src/pgstac/tests/testdata/items.ndjson diff --git a/test/testdata/items.pgcopy b/src/pgstac/tests/testdata/items.pgcopy similarity index 100% rename from test/testdata/items.pgcopy rename to src/pgstac/tests/testdata/items.pgcopy diff --git a/pypgstac/README.md b/src/pypgstac/README.md similarity index 100% rename from pypgstac/README.md rename to src/pypgstac/README.md diff --git a/pypgstac/pypgstac/__init__.py b/src/pypgstac/pypgstac/__init__.py similarity index 100% rename from pypgstac/pypgstac/__init__.py rename to src/pypgstac/pypgstac/__init__.py diff --git a/pypgstac/pypgstac/db.py b/src/pypgstac/pypgstac/db.py similarity index 90% rename from pypgstac/pypgstac/db.py rename to src/pypgstac/pypgstac/db.py index 4c59f209..af0e1499 100644 --- a/pypgstac/pypgstac/db.py +++ b/src/pypgstac/pypgstac/db.py @@ -1,14 +1,15 @@ """Base library for database interaction with PgStac.""" +import atexit +import logging import time from types import TracebackType -from typing import Any, List, Optional, Tuple, Type, Union, Generator +from typing import Any, Generator, List, Optional, Tuple, Type, Union + import orjson import psycopg from psycopg import Connection, sql from psycopg.types.json import set_json_dumps, set_json_loads from psycopg_pool import ConnectionPool -import atexit -import logging from pydantic import BaseSettings from tenacity import retry, retry_if_exception_type, stop_after_attempt @@ -84,10 +85,6 @@ def get_pool(self) -> ConnectionPool: max_waiting=settings.db_max_queries, max_idle=settings.db_max_idle, num_workers=settings.db_num_workers, - kwargs={ - "options": "-c search_path=pgstac,public" - " -c application_name=pypgstac" - }, ) return self.pool @@ -109,6 +106,24 @@ def connect(self) -> Connection: if self.debug: self.connection.add_notice_handler(pg_notice_handler) atexit.register(self.disconnect) + self.connection.execute( + """ + SELECT + CASE + WHEN + current_setting('search_path', false) ~* '\\mpgstac\\M' + THEN current_setting('search_path', false) + ELSE set_config( + 'search_path', + 'pgstac,' || current_setting('search_path', false), + false + ) + END + ; + SET application_name TO 'pgstac'; + """, + prepare=False, + ) return self.connection def wait(self) -> None: @@ -132,12 +147,12 @@ def disconnect(self) -> None: self.connection.commit() if self.connection is not None: self.connection.rollback() - except: + except Exception: pass try: if self.pool is not None and self.connection is not None: self.pool.putconn(self.connection) - except: + except Exception: pass self.connection = None @@ -215,7 +230,7 @@ def version(self) -> Optional[str]: """ SELECT version from pgstac.migrations order by datetime desc, version desc limit 1; - """ + """, ) logger.debug(f"VERSION: {version}") if isinstance(version, bytes): @@ -234,7 +249,7 @@ def pg_version(self) -> str: version = self.query_one( """ SHOW server_version; - """ + """, ) logger.debug(f"PG VERSION: {version}.") if isinstance(version, bytes): diff --git a/pypgstac/pypgstac/hydration.py b/src/pypgstac/pypgstac/hydration.py similarity index 99% rename from pypgstac/pypgstac/hydration.py rename to src/pypgstac/pypgstac/hydration.py index 77d4c480..4d151b1f 100644 --- a/pypgstac/pypgstac/hydration.py +++ b/src/pypgstac/pypgstac/hydration.py @@ -12,7 +12,6 @@ def hydrate(base_item: Dict[str, Any], item: Dict[str, Any]) -> Dict[str, Any]: This will not perform a deep copy; values of the original item will be referenced in the return item. """ - # Merge will mutate i, but create deep copies of values in the base item # This will prevent the base item values from being mutated, e.g. by # filtering out fields in `filter_fields`. diff --git a/pypgstac/pypgstac/load.py b/src/pypgstac/pypgstac/load.py similarity index 92% rename from pypgstac/pypgstac/load.py rename to src/pypgstac/pypgstac/load.py index 58830bdc..883cefc8 100644 --- a/pypgstac/pypgstac/load.py +++ b/src/pypgstac/pypgstac/load.py @@ -1,28 +1,31 @@ """Utilities to bulk load data into pgstac from json/ndjson.""" import contextlib -from datetime import datetime import itertools import logging -from pathlib import Path import sys import time from dataclasses import dataclass -from functools import lru_cache +from datetime import datetime +from enum import Enum +from pathlib import Path from typing import ( Any, BinaryIO, Dict, + Generator, Iterable, Iterator, Optional, + TextIO, Tuple, Union, - Generator, - TextIO, ) + import orjson import psycopg +from cachetools.func import lru_cache from orjson import JSONDecodeError +from pkg_resources import parse_version as V from plpygis.geometry import Geometry from psycopg import sql from psycopg.types.range import Range @@ -34,11 +37,9 @@ wait_random_exponential, ) - from .db import PgstacDB from .hydration import dehydrate from .version import __version__ -from enum import Enum logger = logging.getLogger(__name__) @@ -83,7 +84,7 @@ class Methods(str, Enum): @contextlib.contextmanager def open_std( - filename: str, mode: str = "r", *args: Any, **kwargs: Any + filename: str, mode: str = "r", *args: Any, **kwargs: Any, ) -> Generator[Any, None, None]: """Open files and i/o streams transparently.""" fh: Union[TextIO, BinaryIO] @@ -93,14 +94,8 @@ def open_std( or filename == "stdin" or filename == "stdout" ): - if "r" in mode: - stream = sys.stdin - else: - stream = sys.stdout - if "b" in mode: - fh = stream.buffer - else: - fh = stream + stream = sys.stdin if "r" in mode else sys.stdout + fh = stream.buffer if "b" in mode else stream close = False else: fh = open(filename, mode, *args, **kwargs) @@ -161,10 +156,21 @@ def __init__(self, db: PgstacDB): self._partition_cache: Dict[str, Partition] = {} def check_version(self) -> None: - if self.db.version != __version__: + db_version = self.db.version + if db_version is None: + raise Exception("Failed to detect the target database version.") + + v1 = V(db_version) + v2 = V(__version__) + if (v1.major, v1.minor) != ( + v2.major, + v2.minor, + ): raise Exception( - f"pypgstac version {__version__} is not compatible with the target" + f"pypgstac version {__version__}" + " is not compatible with the target" f" database version {self.db.version}." + f" database version {db_version}.", ) @lru_cache(maxsize=128) @@ -180,7 +186,7 @@ def collection_json(self, collection_id: str) -> Tuple[Dict[str, Any], int, str] raise Exception(f"Error getting info for {collection_id}.") if key is None: raise Exception( - f"Collection {collection_id} is not present in the database" + f"Collection {collection_id} is not present in the database", ) logger.debug(f"Found {collection_id} with base_item {base_item}") return base_item, key, partition_trunc @@ -203,7 +209,7 @@ def load_collections( DROP TABLE IF EXISTS tmp_collections; CREATE TEMP TABLE tmp_collections (content jsonb) ON COMMIT DROP; - """ + """, ) with cur.copy("COPY tmp_collections (content) FROM stdin;") as copy: for collection in read_json(file): @@ -216,7 +222,7 @@ def load_collections( """ INSERT INTO collections (content) SELECT content FROM tmp_collections; - """ + """, ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") @@ -229,7 +235,7 @@ def load_collections( INSERT INTO collections (content) SELECT content FROM tmp_collections ON CONFLICT DO NOTHING; - """ + """, ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") @@ -240,14 +246,14 @@ def load_collections( SELECT content FROM tmp_collections ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content; - """ + """, ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") else: raise Exception( "Available modes are insert, ignore, and upsert." - f"You entered {insert_mode}." + f"You entered {insert_mode}.", ) @retry( @@ -273,20 +279,13 @@ def load_partition( with conn.cursor() as cur: if partition.requires_update: with conn.transaction(): - cur.execute( - "SELECT * FROM partitions WHERE name = %s FOR UPDATE;", - (partition.name,), - ) cur.execute( """ - INSERT INTO partitions - (collection, datetime_range, end_datetime_range) - VALUES - (%s, tstzrange(%s, %s, '[]'), tstzrange(%s,%s, '[]')) - ON CONFLICT (name) DO UPDATE SET - datetime_range = EXCLUDED.datetime_range, - end_datetime_range = EXCLUDED.end_datetime_range - ; + SELECT check_partition( + %s, + tstzrange(%s, %s, '[]'), + tstzrange(%s, %s, '[]') + ); """, ( partition.collection, @@ -296,15 +295,18 @@ def load_partition( partition.end_datetime_range_max, ), ) + logger.debug( f"Adding or updating partition {partition.name} " - f"took {time.perf_counter() - t}s" + f"took {time.perf_counter() - t}s", ) partition.requires_update = False else: logger.debug(f"Partition {partition.name} does not require an update.") with conn.transaction(): + + t = time.perf_counter() if insert_mode in ( None, @@ -316,8 +318,8 @@ def load_partition( COPY {} (id, collection, datetime, end_datetime, geometry, content) FROM stdin; - """ - ).format(sql.Identifier(partition.name)) + """, + ).format(sql.Identifier(partition.name)), ) as copy: for item in items: item.pop("partition") @@ -329,7 +331,7 @@ def load_partition( item["end_datetime"], item["geometry"], item["content"], - ) + ), ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") @@ -344,14 +346,14 @@ def load_partition( DROP TABLE IF EXISTS items_ingest_temp; CREATE TEMP TABLE items_ingest_temp ON COMMIT DROP AS SELECT * FROM items LIMIT 0; - """ + """, ) with cur.copy( """ COPY items_ingest_temp (id, collection, datetime, end_datetime, geometry, content) FROM stdin; - """ + """, ) as copy: for item in items: item.pop("partition") @@ -363,7 +365,7 @@ def load_partition( item["end_datetime"], item["geometry"], item["content"], - ) + ), ) logger.debug(cur.statusmessage) logger.debug(f"Copied rows: {cur.rowcount}") @@ -372,8 +374,8 @@ def load_partition( sql.SQL( """ LOCK TABLE ONLY {} IN EXCLUSIVE MODE; - """ - ).format(sql.Identifier(partition.name)) + """, + ).format(sql.Identifier(partition.name)), ) if insert_mode in ( Methods.ignore, @@ -385,8 +387,8 @@ def load_partition( INSERT INTO {} SELECT * FROM items_ingest_temp ON CONFLICT DO NOTHING; - """ - ).format(sql.Identifier(partition.name)) + """, + ).format(sql.Identifier(partition.name)), ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") @@ -404,8 +406,8 @@ def load_partition( content = EXCLUDED.content WHERE t IS DISTINCT FROM EXCLUDED ; - """ - ).format(sql.Identifier(partition.name)) + """, + ).format(sql.Identifier(partition.name)), ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") @@ -427,18 +429,20 @@ def load_partition( JOIN deletes d USING (id, collection); ; - """ - ).format(sql.Identifier(partition.name)) + """, + ).format(sql.Identifier(partition.name)), + ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") else: raise Exception( "Available modes are insert, ignore, upsert, and delsert." - f"You entered {insert_mode}." + f"You entered {insert_mode}.", ) + cur.execute("SELECT update_partition_stats_q(%s);",(partition.name,)) logger.debug( - f"Copying data for {partition} took {time.perf_counter() - t} seconds" + f"Copying data for {partition} took {time.perf_counter() - t} seconds", ) def _partition_update(self, item: Dict[str, Any]) -> str: @@ -468,10 +472,10 @@ def _partition_update(self, item: Dict[str, Any]) -> str: # Read the partition information from the database if it exists db_rows = list( self.db.query( - "SELECT datetime_range, end_datetime_range " - "FROM partitions WHERE name=%s;", + "SELECT constraint_dtrange, constraint_edtrange " + "FROM partitions WHERE partition=%s;", [partition_name], - ) + ), ) if db_rows: datetime_range: Optional[Range[datetime]] = db_rows[0][0] @@ -562,7 +566,7 @@ def read_dehydrated(self, file: Union[Path, str] = "stdin") -> Generator: yield item def read_hydrated( - self, file: Union[Path, str, Iterator[Any]] = "stdin" + self, file: Union[Path, str, Iterator[Any]] = "stdin", ) -> Generator: for line in read_json(file): item = self.format_item(line) @@ -604,8 +608,8 @@ def format_item(self, _item: Union[Path, str, Dict[str, Any]]) -> Dict[str, Any] if not isinstance(_item, dict): try: item = orjson.loads(str(_item).replace("\\\\", "\\")) - except: - raise Exception(f"Could not load {_item}") + except Exception: + raise else: item = _item @@ -630,7 +634,7 @@ def format_item(self, _item: Union[Path, str, Dict[str, Any]]) -> Dict[str, Any] if out["datetime"] is None or out["end_datetime"] is None: raise Exception( - f"Datetime must be set. OUT: {out} Properties: {properties}" + f"Datetime must be set. OUT: {out} Properties: {properties}", ) if partition_trunc == "year": diff --git a/pypgstac/pypgstac/migrate.py b/src/pypgstac/pypgstac/migrate.py similarity index 97% rename from pypgstac/pypgstac/migrate.py rename to src/pypgstac/pypgstac/migrate.py index cbc43964..19af6698 100644 --- a/pypgstac/pypgstac/migrate.py +++ b/src/pypgstac/pypgstac/migrate.py @@ -1,13 +1,14 @@ """Utilities to help migrate pgstac schema.""" import glob +import logging import os from collections import defaultdict -from typing import Optional, Dict, List, Iterator, Any +from typing import Any, Dict, Iterator, List, Optional + from smart_open import open -import logging -from .db import PgstacDB from . import __version__ +from .db import PgstacDB dirname = os.path.dirname(__file__) migrations_dir = os.path.join(dirname, "migrations") @@ -34,7 +35,7 @@ def __init__(self, path: str, f: str, t: str) -> None: def parse_filename(self, filename: str) -> List[str]: """Get version numbers from filename.""" filename = os.path.splitext(os.path.basename(filename))[0].replace( - "pgstac.", "" + "pgstac.", "", ) return filename.split("-") @@ -79,7 +80,7 @@ def migrations(self) -> List[str]: path = self.build_path() if path is None: raise Exception( - f"Could not determine path to get from {self.f} to {self.t}." + f"Could not determine path to get from {self.f} to {self.t}.", ) if len(path) == 1: return [f"pgstac.{path[0]}.sql"] @@ -152,7 +153,7 @@ def run_migration(self, toversion: Optional[str] = None) -> str: else: conn.rollback() raise Exception( - "Migration failed, database rolled back to previous state." + "Migration failed, database rolled back to previous state.", ) logger.debug(f"New Version: {newversion}") diff --git a/src/pypgstac/pypgstac/migrations b/src/pypgstac/pypgstac/migrations new file mode 120000 index 00000000..6fef8af1 --- /dev/null +++ b/src/pypgstac/pypgstac/migrations @@ -0,0 +1 @@ +../../pgstac/migrations \ No newline at end of file diff --git a/pypgstac/pypgstac/py.typed b/src/pypgstac/pypgstac/py.typed similarity index 100% rename from pypgstac/pypgstac/py.typed rename to src/pypgstac/pypgstac/py.typed diff --git a/pypgstac/pypgstac/pypgstac.py b/src/pypgstac/pypgstac/pypgstac.py similarity index 91% rename from pypgstac/pypgstac/pypgstac.py rename to src/pypgstac/pypgstac/pypgstac.py index 4180c73d..b94c3ec9 100755 --- a/pypgstac/pypgstac/pypgstac.py +++ b/src/pypgstac/pypgstac/pypgstac.py @@ -1,27 +1,25 @@ """Command utilities for managing pgstac.""" -from typing import Optional +import logging import sys +from typing import Optional + import fire -from pypgstac.db import PgstacDB -from pypgstac.migrate import Migrate -from pypgstac.load import Loader, Methods, Tables -from pypgstac.version import __version__ -import logging from smart_open import open -# sys.tracebacklimit = 0 +from pypgstac.db import PgstacDB +from pypgstac.load import Loader, Methods, Tables +from pypgstac.migrate import Migrate class PgstacCLI: """CLI for PgStac.""" def __init__( - self, dsn: Optional[str] = "", version: bool = False, debug: bool = False + self, dsn: Optional[str] = "", version: bool = False, debug: bool = False, ): """Initialize PgStac CLI.""" if version: - print(__version__) sys.exit(0) self.dsn = dsn @@ -86,19 +84,18 @@ def loadextensions(self) -> None: ) FROM collections ON CONFLICT DO NOTHING; - """ + """, ) conn.commit() urls = self._db.query( """ SELECT url FROM stac_extensions WHERE content IS NULL; - """ + """, ) if urls: for u in urls: url = u[0] - print(f"Fetching content from {url}") try: with open(url, "r") as f: content = f.read() @@ -112,8 +109,8 @@ def loadextensions(self) -> None: [content, url], ) conn.commit() - except Exception as e: - print(f"Unable to load {url} into pgstac. {e}") + except Exception: + pass def cli() -> fire.Fire: diff --git a/src/pypgstac/pypgstac/version.py b/src/pypgstac/pypgstac/version.py new file mode 100644 index 00000000..111eb5eb --- /dev/null +++ b/src/pypgstac/pypgstac/version.py @@ -0,0 +1,2 @@ +"""Version.""" +__version__ = "0.7.0" diff --git a/src/pypgstac/pyproject.toml b/src/pypgstac/pyproject.toml new file mode 100644 index 00000000..b557a61b --- /dev/null +++ b/src/pypgstac/pyproject.toml @@ -0,0 +1,134 @@ +[project] +name = "pypgstac" +description = "Schema, functions and a python library for storing and accessing STAC collections and items in PostgreSQL" +readme = "README.md" +requires-python = ">=3.8" +license = "MIT" +authors = [ + {name = "David Bitner", email = "bitner@dbspatial.com"}, +] +keywords = ["STAC", "Postgresql", "PgSTAC"] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +dynamic = ["version"] +dependencies = [ + "smart-open>=4.2,<7.0", + "orjson>=3.5.2", + "python-dateutil==2.8.*", + "fire==0.4.*", + "plpygis==0.2.*", + "pydantic[dotenv]==1.10.*", + "tenacity==8.1.*", + "cachetools==5.3.*", +] + +[project.optional-dependencies] +test = [ + "pytest", + "pytest-cov", + "pystac[validation]==1.*", + "types-cachetools", +] +dev = [ + "flake8==3.9.*", + "black>=21.7b0", + "mypy>=0.910", + "types-orjson==0.1.1", + "types-pkg-resources", + "ruff==0.0.231", + "pre-commit", + "psycopg2-binary", + "migra", +] +psycopg = [ + "psycopg[binary]==3.1.*", + "psycopg-pool==3.1.*", +] + + +[project.urls] +Homepage = "https://stac-utils.github.io/pgstac/" +Documentation = "https://stac-utils.github.io/pgstac/" +Issues = "https://github.com/stac-utils/pgstac/issues" +Source = "https://github.com/stac-utils/pgstac" +Changelog = "https://stac-utils.github.io/pgstac/release-notes/" + +[project.scripts] +pypgstac = "pypgstac.pypgstac:cli" + + +[tool.hatch.version] +path = "pypgstac/version.py" + +[tool.hatch.build.targets.sdist] +exclude = [ + "/tests", + "/docs", + ".pytest_cache", + ".gitignore", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.coverage.run] +branch = true +parallel = true + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.ruff] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "C", # flake8-comprehensions + "B", # flake8-bugbear + # "D", # pydocstyle + "C4", # flake8-comprehensions + "T20", # flake8-print + # "PT", # flake8-pytest-style + "Q", # flake8-quotes + # "SIM", # flake8-simplify + "DTZ", # flake8-datetimez + "ERA", # eradicate + "PLC", + "PLE", + # "PLR", + "PLW", + "COM", # flake8-commas +] +ignore = [ + # "E501", # line too long, handled by black + "B008", # do not perform function calls in argument defaults + "C901", # too complex + "B905", +] + +[tool.ruff.isort] +known-first-party = ["pypgstac"] + +[tool.mypy] +no_strict_optional = "True" +ignore_missing_imports = "True" +disallow_untyped_defs = "True" +namespace_packages = "True" + +[tool.pydocstyle] +select = "D1" +match = "(?!test).*.py" diff --git a/src/pypgstac/setup.py b/src/pypgstac/setup.py new file mode 100644 index 00000000..8474b7d2 --- /dev/null +++ b/src/pypgstac/setup.py @@ -0,0 +1,35 @@ +"""Fake pypgstac setup.py for github.""" +import sys + +from setuptools import setup + +sys.stderr.write( + """ +=============================== +Unsupported installation method +=============================== +pypgstac no longer supports installation with `python setup.py install`. +Please use `python -m pip install .` instead. +""", +) +sys.exit(1) + + +# The below code will never execute, however GitHub is particularly +# picky about where it finds Python packaging metadata. +# See: https://github.com/github/feedback/discussions/6456 +# +# To be removed once GitHub catches up. + +setup( + name="pypgstac", + install_requires=[ + "smart-open>=4.2,<7.0", + "orjson>=3.5.2", + "python-dateutil==2.8.*", + "fire==0.4.*", + "plpygis==0.2.*", + "pydantic[dotenv]==1.10.*", + "tenacity==8.1.*", + ], +) diff --git a/pypgstac/tests/__init__.py b/src/pypgstac/tests/__init__.py similarity index 100% rename from pypgstac/tests/__init__.py rename to src/pypgstac/tests/__init__.py diff --git a/src/pypgstac/tests/conftest.py b/src/pypgstac/tests/conftest.py new file mode 100644 index 00000000..d7536799 --- /dev/null +++ b/src/pypgstac/tests/conftest.py @@ -0,0 +1,77 @@ +"""Fixtures for pypgstac tests.""" +import os +from typing import Generator + +import psycopg +import pytest + +from pypgstac.db import PgstacDB +from pypgstac.load import Loader +from pypgstac.migrate import Migrate + + +@pytest.fixture(scope="function") +def db() -> Generator: + """Fixture to get a fresh database.""" + origdb: str = os.getenv("PGDATABASE", "") + + with psycopg.connect(autocommit=True) as conn: + try: + conn.execute( + """ + CREATE DATABASE pypgstactestdb + TEMPLATE pgstac_test_db_template; + """, + ) + except psycopg.errors.DuplicateDatabase: + try: + conn.execute( + """ + DROP DATABASE pypgstactestdb WITH (FORCE); + """, + ) + conn.execute( + """ + CREATE DATABASE pypgstactestdb + TEMPLATE pgstac_test_db_template; + """, + ) + except psycopg.errors.InsufficientPrivilege: + try: + conn.execute("DROP DATABASE pypgstactestdb;") + conn.execute( + """ + CREATE DATABASE pypgstactestdb + TEMPLATE pgstac_test_db_template; + """, + ) + except Exception: + pass + + os.environ["PGDATABASE"] = "pypgstactestdb" + + pgdb = PgstacDB() + + yield pgdb + + pgdb.close() + os.environ["PGDATABASE"] = origdb + + with psycopg.connect(autocommit=True) as conn: + try: + conn.execute("DROP DATABASE pypgstactestdb WITH (FORCE);") + except psycopg.errors.InsufficientPrivilege: + try: + conn.execute("DROP DATABASE pypgstactestdb;") + except Exception: + pass + + +@pytest.fixture(scope="function") +def loader(db: PgstacDB) -> Generator: + """Fixture to get a loader and an empty pgstac.""" + if False: + db.query("DROP SCHEMA IF EXISTS pgstac CASCADE;") + Migrate(db).run_migration() + ldr = Loader(db) + return ldr diff --git a/pypgstac/tests/data-files/hydration/collections/chloris-biomass.json b/src/pypgstac/tests/data-files/hydration/collections/chloris-biomass.json similarity index 100% rename from pypgstac/tests/data-files/hydration/collections/chloris-biomass.json rename to src/pypgstac/tests/data-files/hydration/collections/chloris-biomass.json diff --git a/pypgstac/tests/data-files/hydration/collections/landsat-c2-l1.json b/src/pypgstac/tests/data-files/hydration/collections/landsat-c2-l1.json similarity index 100% rename from pypgstac/tests/data-files/hydration/collections/landsat-c2-l1.json rename to src/pypgstac/tests/data-files/hydration/collections/landsat-c2-l1.json diff --git a/pypgstac/tests/data-files/hydration/collections/sentinel-1-grd.json b/src/pypgstac/tests/data-files/hydration/collections/sentinel-1-grd.json similarity index 100% rename from pypgstac/tests/data-files/hydration/collections/sentinel-1-grd.json rename to src/pypgstac/tests/data-files/hydration/collections/sentinel-1-grd.json diff --git a/pypgstac/tests/data-files/hydration/dehydrated-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json b/src/pypgstac/tests/data-files/hydration/dehydrated-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json similarity index 100% rename from pypgstac/tests/data-files/hydration/dehydrated-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json rename to src/pypgstac/tests/data-files/hydration/dehydrated-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json diff --git a/pypgstac/tests/data-files/hydration/raw-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json b/src/pypgstac/tests/data-files/hydration/raw-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json similarity index 100% rename from pypgstac/tests/data-files/hydration/raw-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json rename to src/pypgstac/tests/data-files/hydration/raw-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json diff --git a/pypgstac/tests/data-files/hydration/raw-items/sentinel-1-grd/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C.json b/src/pypgstac/tests/data-files/hydration/raw-items/sentinel-1-grd/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C.json similarity index 100% rename from pypgstac/tests/data-files/hydration/raw-items/sentinel-1-grd/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C.json rename to src/pypgstac/tests/data-files/hydration/raw-items/sentinel-1-grd/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C.json diff --git a/pypgstac/tests/data-files/load/dehydrated.txt b/src/pypgstac/tests/data-files/load/dehydrated.txt similarity index 100% rename from pypgstac/tests/data-files/load/dehydrated.txt rename to src/pypgstac/tests/data-files/load/dehydrated.txt diff --git a/pypgstac/tests/hydration/__init__.py b/src/pypgstac/tests/hydration/__init__.py similarity index 100% rename from pypgstac/tests/hydration/__init__.py rename to src/pypgstac/tests/hydration/__init__.py diff --git a/pypgstac/tests/hydration/test_base_item.py b/src/pypgstac/tests/hydration/test_base_item.py similarity index 90% rename from pypgstac/tests/hydration/test_base_item.py rename to src/pypgstac/tests/hydration/test_base_item.py index 87271bfa..8c360fb1 100644 --- a/pypgstac/tests/hydration/test_base_item.py +++ b/src/pypgstac/tests/hydration/test_base_item.py @@ -4,7 +4,6 @@ from pypgstac.load import Loader - HERE = Path(__file__).parent LANDSAT_COLLECTION = ( HERE / ".." / "data-files" / "hydration" / "collections" / "landsat-c2-l1.json" @@ -13,7 +12,9 @@ def test_landsat_c2_l1(loader: Loader) -> None: """Test that a base item is created when a collection is loaded and that it - is equal to the item_assets of the collection""" + is equal to the item_assets of the collection + . + """ with open(LANDSAT_COLLECTION) as f: collection = json.load(f) loader.load_collections(str(LANDSAT_COLLECTION)) @@ -21,7 +22,7 @@ def test_landsat_c2_l1(loader: Loader) -> None: base_item = cast( Dict[str, Any], loader.db.query_one( - "SELECT base_item FROM collections WHERE id=%s;", (collection["id"],) + "SELECT base_item FROM collections WHERE id=%s;", (collection["id"],), ), ) diff --git a/pypgstac/tests/hydration/test_dehydrate.py b/src/pypgstac/tests/hydration/test_dehydrate.py similarity index 97% rename from pypgstac/tests/hydration/test_dehydrate.py rename to src/pypgstac/tests/hydration/test_dehydrate.py index 44479ac9..fe837f0b 100644 --- a/pypgstac/tests/hydration/test_dehydrate.py +++ b/src/pypgstac/tests/hydration/test_dehydrate.py @@ -6,7 +6,6 @@ from pypgstac.hydration import DO_NOT_MERGE_MARKER from pypgstac.load import Loader - HERE = Path(__file__).parent LANDSAT_COLLECTION = ( HERE / ".." / "data-files" / "hydration" / "collections" / "landsat-c2-l1.json" @@ -24,14 +23,14 @@ class TestDehydrate: def dehydrate( - self, base_item: Dict[str, Any], item: Dict[str, Any] + self, base_item: Dict[str, Any], item: Dict[str, Any], ) -> Dict[str, Any]: return hydration.dehydrate(base_item, item) def test_landsat_c2_l1(self, loader: Loader) -> None: """ Test that a dehydrated item is created properly from a raw item against a - base item from a collection + base item from a collection. """ with open(LANDSAT_COLLECTION) as f: collection = json.load(f) @@ -43,7 +42,7 @@ def test_landsat_c2_l1(self, loader: Loader) -> None: base_item = cast( Dict[str, Any], loader.db.query_one( - "SELECT base_item FROM collections WHERE id=%s;", (collection["id"],) + "SELECT base_item FROM collections WHERE id=%s;", (collection["id"],), ), ) @@ -77,7 +76,7 @@ def test_landsat_c2_l1(self, loader: Loader) -> None: assert list(red["raster:bands"][0].keys()) == ["scale", "offset"] item_red_rb = item["assets"]["red"]["raster:bands"][0] assert red["raster:bands"] == [ - {"scale": item_red_rb["scale"], "offset": item_red_rb["offset"]} + {"scale": item_red_rb["scale"], "offset": item_red_rb["offset"]}, ] # nir09 asset raster bands does not have a `unit` attribute, which is @@ -115,7 +114,7 @@ def test_nested_extra_keys(self) -> None: assert dehydrated == {"c": {"e": "fourth", "f": "fifth"}} def test_list_of_dicts_extra_keys(self) -> None: - """Test that an equal length list of dicts is dehydrated correctly""" + """Test that an equal length list of dicts is dehydrated correctly.""" base_item = {"a": [{"b1": 1, "b2": 2}, {"c1": 1, "c2": 2}]} item = {"a": [{"b1": 1, "b2": 2, "b3": 3}, {"c1": 1, "c2": 2, "c3": 3}]} @@ -136,7 +135,7 @@ def test_equal_len_list_of_mixed_types(self) -> None: "far", {"c1": 1, "c2": 2, "c3": 3}, "boo", - ] + ], } dehydrated = self.dehydrate(base_item, item) @@ -144,7 +143,7 @@ def test_equal_len_list_of_mixed_types(self) -> None: assert dehydrated["a"] == [{"b3": 3}, "far", {"c3": 3}, "boo"] def test_unequal_len_list(self) -> None: - """Test that unequal length lists preserve the item value exactly""" + """Test that unequal length lists preserve the item value exactly.""" base_item = {"a": [{"b1": 1}, {"c1": 1}, {"d1": 1}]} item = {"a": [{"b1": 1, "b2": 2}, {"c1": 1, "c2": 2}]} @@ -164,7 +163,7 @@ def test_marked_non_merged_fields(self) -> None: def test_marked_non_merged_fields_in_list(self) -> None: base_item = { - "a": [{"b": "first", "d": "third"}, {"c": "second", "e": "fourth"}] + "a": [{"b": "first", "d": "third"}, {"c": "second", "e": "fourth"}], } item = {"a": [{"b": "first"}, {"c": "second", "f": "fifth"}]} @@ -173,7 +172,7 @@ def test_marked_non_merged_fields_in_list(self) -> None: "a": [ {"d": DO_NOT_MERGE_MARKER}, {"e": DO_NOT_MERGE_MARKER, "f": "fifth"}, - ] + ], } def test_deeply_nested_dict(self) -> None: @@ -184,10 +183,10 @@ def test_deeply_nested_dict(self) -> None: assert dehydrated == {"a": {"b": {"c": {"d2": "third"}}}} def test_equal_list_of_non_dicts(self) -> None: - """Values of lists that match base_item should be dehydrated off""" + """Values of lists that match base_item should be dehydrated off.""" base_item = {"assets": {"thumbnail": {"roles": ["thumbnail"]}}} item = { - "assets": {"thumbnail": {"roles": ["thumbnail"], "href": "http://foo.com"}} + "assets": {"thumbnail": {"roles": ["thumbnail"], "href": "http://foo.com"}}, } dehydrated = self.dehydrate(base_item, item) @@ -207,7 +206,7 @@ def test_invalid_assets_marked(self) -> None: }, } hydrated = { - "assets": {"asset1": {"name": "Asset one", "href": "http://foo.com"}} + "assets": {"asset1": {"name": "Asset one", "href": "http://foo.com"}}, } dehydrated = self.dehydrate(base_item, hydrated) diff --git a/pypgstac/tests/hydration/test_dehydrate_pg.py b/src/pypgstac/tests/hydration/test_dehydrate_pg.py similarity index 79% rename from pypgstac/tests/hydration/test_dehydrate_pg.py rename to src/pypgstac/tests/hydration/test_dehydrate_pg.py index a19ba516..24b23a66 100644 --- a/pypgstac/tests/hydration/test_dehydrate_pg.py +++ b/src/pypgstac/tests/hydration/test_dehydrate_pg.py @@ -1,11 +1,13 @@ -from .test_dehydrate import TestDehydrate as TDehydrate -from typing import Dict, Any import os -from typing import Generator +from contextlib import contextmanager +from typing import Any, Dict, Generator + +import psycopg + from pypgstac.db import PgstacDB from pypgstac.migrate import Migrate -import psycopg -from contextlib import contextmanager + +from .test_dehydrate import TestDehydrate as TDehydrate class TestDehydratePG(TDehydrate): @@ -14,7 +16,6 @@ class TestDehydratePG(TDehydrate): @contextmanager def db(self) -> Generator: """Set up database connection.""" - print("Setting up db.") origdb: str = os.getenv("PGDATABASE", "") with psycopg.connect(autocommit=True) as conn: try: @@ -26,17 +27,15 @@ def db(self) -> Generator: pgdb = PgstacDB() pgdb.query("DROP SCHEMA IF EXISTS pgstac CASCADE;") - migrator = Migrate(pgdb) - print(migrator.run_migration()) + Migrate(pgdb).run_migration() yield pgdb - print("Closing Connection to DB") pgdb.close() os.environ["PGDATABASE"] = origdb def dehydrate( - self, base_item: Dict[str, Any], item: Dict[str, Any] + self, base_item: Dict[str, Any], item: Dict[str, Any], ) -> Dict[str, Any]: """Dehydrate item using pgstac.""" with self.db() as db: diff --git a/pypgstac/tests/hydration/test_hydrate.py b/src/pypgstac/tests/hydration/test_hydrate.py similarity index 95% rename from pypgstac/tests/hydration/test_hydrate.py rename to src/pypgstac/tests/hydration/test_hydrate.py index 01fc2350..861b1f62 100644 --- a/pypgstac/tests/hydration/test_hydrate.py +++ b/src/pypgstac/tests/hydration/test_hydrate.py @@ -7,7 +7,6 @@ from pypgstac.hydration import DO_NOT_MERGE_MARKER from pypgstac.load import Loader - HERE = Path(__file__).parent LANDSAT_COLLECTION = ( HERE / ".." / "data-files" / "hydration" / "collections" / "landsat-c2-l1.json" @@ -35,13 +34,15 @@ class TestHydrate: def hydrate( - self, base_item: Dict[str, Any], item: Dict[str, Any] + self, base_item: Dict[str, Any], item: Dict[str, Any], ) -> Dict[str, Any]: return hydration.hydrate(base_item, item) def test_landsat_c2_l1(self, loader: Loader) -> None: """Test that a dehydrated item is is equal to the raw item it was dehydrated - from, against the base item of the collection""" + from, against the base item of the collection + . + """ with open(LANDSAT_COLLECTION) as f: collection = json.load(f) loader.load_collections(str(LANDSAT_COLLECTION)) @@ -95,13 +96,13 @@ def test_nested_extra_keys(self) -> None: } def test_list_of_dicts_extra_keys(self) -> None: - """Test that an equal length list of dicts is hydrated correctly""" + """Test that an equal length list of dicts is hydrated correctly.""" base_item = {"a": [{"b1": 1, "b2": 2}, {"c1": 1, "c2": 2}]} dehydrated = {"a": [{"b3": 3}, {"c3": 3}]} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { - "a": [{"b1": 1, "b2": 2, "b3": 3}, {"c1": 1, "c2": 2, "c3": 3}] + "a": [{"b1": 1, "b2": 2, "b3": 3}, {"c1": 1, "c2": 2, "c3": 3}], } def test_equal_len_list_of_mixed_types(self) -> None: @@ -120,11 +121,11 @@ def test_equal_len_list_of_mixed_types(self) -> None: "far", {"c1": 1, "c2": 2, "c3": 3}, "boo", - ] + ], } def test_unequal_len_list(self) -> None: - """Test that unequal length lists preserve the item value exactly""" + """Test that unequal length lists preserve the item value exactly.""" base_item = {"a": [{"b1": 1}, {"c1": 1}, {"d1": 1}]} dehydrated = {"a": [{"b1": 1, "b2": 2}, {"c1": 1, "c2": 2}]} @@ -148,13 +149,13 @@ def test_marked_non_merged_fields(self) -> None: def test_marked_non_merged_fields_in_list(self) -> None: base_item = { - "a": [{"b": "first", "d": "third"}, {"c": "second", "e": "fourth"}] + "a": [{"b": "first", "d": "third"}, {"c": "second", "e": "fourth"}], } dehydrated = { "a": [ {"d": DO_NOT_MERGE_MARKER}, {"e": DO_NOT_MERGE_MARKER, "f": "fifth"}, - ] + ], } hydrated = self.hydrate(base_item, dehydrated) @@ -166,24 +167,24 @@ def test_deeply_nested_dict(self) -> None: hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { - "a": {"b": {"c": {"d": "first", "d1": "second", "d2": "third"}}} + "a": {"b": {"c": {"d": "first", "d1": "second", "d2": "third"}}}, } def test_equal_list_of_non_dicts(self) -> None: - """Values of lists that match base_item should be hydrated back on""" + """Values of lists that match base_item should be hydrated back on.""" base_item = {"assets": {"thumbnail": {"roles": ["thumbnail"]}}} dehydrated = {"assets": {"thumbnail": {"href": "http://foo.com"}}} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { - "assets": {"thumbnail": {"roles": ["thumbnail"], "href": "http://foo.com"}} + "assets": {"thumbnail": {"roles": ["thumbnail"], "href": "http://foo.com"}}, } def test_invalid_assets_removed(self) -> None: """ Assets can be included on item-assets that are not uniformly included on individual items. Ensure that item asset keys from base_item aren't included - after hydration + after hydration. """ base_item = { "type": "Feature", diff --git a/pypgstac/tests/hydration/test_hydrate_pg.py b/src/pypgstac/tests/hydration/test_hydrate_pg.py similarity index 79% rename from pypgstac/tests/hydration/test_hydrate_pg.py rename to src/pypgstac/tests/hydration/test_hydrate_pg.py index 626de958..1245e37a 100644 --- a/pypgstac/tests/hydration/test_hydrate_pg.py +++ b/src/pypgstac/tests/hydration/test_hydrate_pg.py @@ -1,12 +1,14 @@ """Test Hydration in PGStac.""" -from .test_hydrate import TestHydrate as THydrate -from typing import Dict, Any import os -from typing import Generator +from contextlib import contextmanager +from typing import Any, Dict, Generator + +import psycopg + from pypgstac.db import PgstacDB from pypgstac.migrate import Migrate -import psycopg -from contextlib import contextmanager + +from .test_hydrate import TestHydrate as THydrate class TestHydratePG(THydrate): @@ -15,7 +17,6 @@ class TestHydratePG(THydrate): @contextmanager def db(self) -> Generator[PgstacDB, None, None]: """Set up database.""" - print("Setting up db.") origdb: str = os.getenv("PGDATABASE", "") with psycopg.connect(autocommit=True) as conn: try: @@ -27,17 +28,15 @@ def db(self) -> Generator[PgstacDB, None, None]: pgdb = PgstacDB() pgdb.query("DROP SCHEMA IF EXISTS pgstac CASCADE;") - migrator = Migrate(pgdb) - print(migrator.run_migration()) + Migrate(pgdb).run_migration() yield pgdb - print("Closing Connection to DB") pgdb.close() os.environ["PGDATABASE"] = origdb def hydrate( - self, base_item: Dict[str, Any], item: Dict[str, Any] + self, base_item: Dict[str, Any], item: Dict[str, Any], ) -> Dict[str, Any]: """Hydrate using pgstac.""" with self.db() as db: diff --git a/pypgstac/tests/test_load.py b/src/pypgstac/tests/test_load.py similarity index 89% rename from pypgstac/tests/test_load.py rename to src/pypgstac/tests/test_load.py index 204e97f1..9084612e 100644 --- a/pypgstac/tests/test_load.py +++ b/src/pypgstac/tests/test_load.py @@ -2,13 +2,16 @@ import json from pathlib import Path from unittest import mock -from pypgstac.load import Methods, Loader, read_json -from psycopg.errors import UniqueViolation -import pytest + import pystac +import pytest +from pkg_resources import parse_version as V +from psycopg.errors import UniqueViolation + +from pypgstac.load import Loader, Methods, __version__, read_json HERE = Path(__file__).parent -TEST_DATA_DIR = HERE.parent.parent / "test" / "testdata" +TEST_DATA_DIR = HERE.parent.parent / "pgstac" / "tests" / "testdata" TEST_COLLECTIONS_JSON = TEST_DATA_DIR / "collections.json" TEST_COLLECTIONS = TEST_DATA_DIR / "collections.ndjson" TEST_ITEMS = TEST_DATA_DIR / "items.ndjson" @@ -27,6 +30,9 @@ / "S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C.json" ) +def version_increment(source_version: str) -> str: + version = V(source_version) + return ".".join(map(str, [version.major, version.minor, version.micro + 1])) def test_load_collections_succeeds(loader: Loader) -> None: """Test pypgstac collections loader.""" @@ -181,7 +187,7 @@ def test_partition_loads_default(loader: Loader) -> None: partitions = loader.db.query_one( """ SELECT count(*) from partitions; - """ + """, ) assert partitions == 1 @@ -197,7 +203,7 @@ def test_partition_loads_month(loader: Loader) -> None: loader.db.connection.execute( """ UPDATE collections SET partition_trunc='month'; - """ + """, ) loader.load_items( @@ -208,7 +214,7 @@ def test_partition_loads_month(loader: Loader) -> None: partitions = loader.db.query_one( """ SELECT count(*) from partitions; - """ + """, ) assert partitions == 2 @@ -224,7 +230,7 @@ def test_partition_loads_year(loader: Loader) -> None: loader.db.connection.execute( """ UPDATE collections SET partition_trunc='year'; - """ + """, ) loader.load_items( @@ -235,7 +241,7 @@ def test_partition_loads_year(loader: Loader) -> None: partitions = loader.db.query_one( """ SELECT count(*) from partitions; - """ + """, ) assert partitions == 1 @@ -249,11 +255,11 @@ def test_load_items_dehydrated_ignore_succeeds(loader: Loader) -> None: ) loader.load_items( - str(TEST_DEHYDRATED_ITEMS), insert_mode=Methods.insert, dehydrated=True + str(TEST_DEHYDRATED_ITEMS), insert_mode=Methods.insert, dehydrated=True, ) loader.load_items( - str(TEST_DEHYDRATED_ITEMS), insert_mode=Methods.ignore, dehydrated=True + str(TEST_DEHYDRATED_ITEMS), insert_mode=Methods.ignore, dehydrated=True, ) @@ -317,7 +323,7 @@ def test_s1_grd_load_and_query(loader: Loader) -> None: loader.db.func( "search", search_body, - ) + ), )[0] item = res["features"][0] pystac.Item.from_dict(item).validate() @@ -338,14 +344,14 @@ def test_load_dehydrated(loader: Loader) -> None: dehydrated_items = HERE / "data-files" / "load" / "dehydrated.txt" loader.load_items( - str(dehydrated_items), insert_mode=Methods.insert, dehydrated=True + str(dehydrated_items), insert_mode=Methods.insert, dehydrated=True, ) def test_load_collections_incompatible_version(loader: Loader) -> None: """Test pypgstac collections loader raises an exception for incompatible version.""" with mock.patch( - "pypgstac.db.PgstacDB.version", new_callable=mock.PropertyMock + "pypgstac.db.PgstacDB.version", new_callable=mock.PropertyMock, ) as mock_version: mock_version.return_value = "dummy" with pytest.raises(Exception): @@ -357,8 +363,12 @@ def test_load_collections_incompatible_version(loader: Loader) -> None: def test_load_items_incompatible_version(loader: Loader) -> None: """Test pypgstac items loader raises an exception for incompatible version.""" + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.insert, + ) with mock.patch( - "pypgstac.db.PgstacDB.version", new_callable=mock.PropertyMock + "pypgstac.db.PgstacDB.version", new_callable=mock.PropertyMock, ) as mock_version: mock_version.return_value = "dummy" with pytest.raises(Exception): @@ -366,3 +376,19 @@ def test_load_items_incompatible_version(loader: Loader) -> None: str(TEST_ITEMS), insert_mode=Methods.insert, ) + + +def test_load_compatible_major_minor_version(loader: Loader) -> None: + """Test pypgstac loader doesn't raise an exception.""" + with mock.patch( + "pypgstac.load.__version__", version_increment(__version__), + ) as mock_version: + loader.load_collections( + str(TEST_COLLECTIONS_JSON), + insert_mode=Methods.insert, + ) + loader.load_items( + str(TEST_ITEMS), + insert_mode=Methods.insert, + ) + assert mock_version != loader.db.version diff --git a/test/basic/sqltest.sh b/test/basic/sqltest.sh deleted file mode 100755 index 39ba57d7..00000000 --- a/test/basic/sqltest.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -SCRIPTDIR=$(dirname "$0") -cd $SCRIPTDIR -SQLFILE=$(pwd)/$1 -SQLOUTFILE=${SQLFILE}.out -PGDATABASE_OLD=$PGDATABASE - -echo $SQLFILE -echo $SQLOUTFILE - -psql <"$TMPFILE" -\set QUIET 1 -\set ON_ERROR_STOP 1 -\set ON_ERROR_ROLLBACK 1 - -BEGIN; -SET SEARCH_PATH TO pgstac, public; -SET client_min_messages TO 'warning'; -SET pgstac.context TO 'on'; -SET pgstac."default_filter_lang" TO 'cql-json'; - -DELETE FROM collections WHERE id = 'pgstac-test-collection'; -\copy collections (content) FROM '../testdata/collections.ndjson'; -\copy items_staging (content) FROM '../testdata/items.ndjson' - -\t - -\set QUIET 0 -\set ECHO all -$(cat $SQLFILE) -\set QUIET 1 -\set ECHO none -ROLLBACK; -EOSQL - -if [ "$2" == "generateout" ]; then - echo "Creating $SQLOUTFILE" - cat $TMPFILE >$SQLOUTFILE -else - diff -Z -b -w -B --strip-trailing-cr "$TMPFILE" $SQLOUTFILE - error=$? -fi - -export PGDATABASE=$PGDATABASE_OLD -psql <