Skip to content

Commit

Permalink
HOTFIX: Patch to syncstores to enable support for PostGIS 3.4 (#1095)
Browse files Browse the repository at this point in the history 
* Add postgis_raster initialization (for PostGIS 3.4)

* Remove close

* Syncstores enables postgis raster extension for PostGIS 3.X or higher

* Conditionally enable postgis raster for postgis version 3+ functional

* black foratting

* Add tests for new postgis 3 functionality

* Fix docker build for 4.2
  • Loading branch information
@swainn
swainn authored Sep 26, 2024
1 parent a7957a6 commit 25dc4a3
Show file tree
Hide file tree
Showing 4 changed files with 276 additions and 16 deletions.
9 changes: 5 additions & 4 deletions .github/workflows/tethys-release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ on:

env:
CONDA_BUILD_PIN_LEVEL: minor
DOCKER_UPLOAD_URL: tethysplatform/tethys-core

jobs:
docker-build:
Expand Down Expand Up @@ -58,7 +59,7 @@ jobs:
full_tag="${{ steps.version.outputs.full }}";
# no "+" characters allowed in Docker tags
safe_tag="${full_tag//+/-}";
docker build -t ${{ secrets.DOCKER_UPLOAD_URL }}:"${{ steps.safetag.outputs.safetag }}" .;
docker build -t ${{env.DOCKER_UPLOAD_URL }}:"${{ steps.safetag.outputs.safetag }}" .;
# Authenticate docker
- name: Authenticate Docker
run: |
Expand All @@ -67,13 +68,13 @@ jobs:
- name: Upload Docker With Tag
run: |
echo "Pushing to docker registry";
docker push ${{ secrets.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }};
docker push ${{ env.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }};
- name: Upload Docker With Latest Tag
if: ${{ steps.version.outputs.prerelease == '' }}
run: |
echo "Updating latest on the docker registry";
docker tag ${{ secrets.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }} ${{ secrets.DOCKER_UPLOAD_URL }}:latest;
docker push ${{ secrets.DOCKER_UPLOAD_URL }}:latest;
docker tag ${{ env.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }} ${{env.DOCKER_UPLOAD_URL }}:latest;
docker push ${{ env.DOCKER_UPLOAD_URL }}:latest;
conda-build:
name: Conda Build (${{ matrix.platform }})
Expand Down
7 changes: 4 additions & 3 deletions .github/workflows/tethys.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ on:

env:
CONDA_BUILD_PIN_LEVEL: minor
DOCKER_UPLOAD_URL: tethysplatform/tethys-core
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TEST_IMAGE: tethys:dev
POSTGRES_DB: tethys_postgis
Expand Down Expand Up @@ -107,15 +108,15 @@ jobs:
# Build the docker for no tag
- name: Build Without Tag
run: |
docker build -t ${{ secrets.DOCKER_UPLOAD_URL }}:dev .;
docker tag ${{ secrets.DOCKER_UPLOAD_URL }}:dev ${{ env.TEST_IMAGE }};
docker build -t ${{ env.DOCKER_UPLOAD_URL }}:4.2-dev .;
docker tag ${{ env.DOCKER_UPLOAD_URL }}:4.2-dev ${{ env.TEST_IMAGE }};
# Upload docker if pull request no tag
- name: Upload Docker No Tag
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "Pushing to docker registry";
echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin;
docker push ${{ secrets.DOCKER_UPLOAD_URL }}:dev;
docker push ${{ env.DOCKER_UPLOAD_URL }}:4.2-dev;
# No Upload if Pull Request
- name: No Upload
if: ${{ github.event_name == 'pull_request' }}
Expand Down
239 changes: 235 additions & 4 deletions tests/unit_tests/test_tethys_apps/test_models/test_PersistentStoreDatabaseSetting.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,16 +427,247 @@ def test_create_persistent_store_database(
).create_persistent_store_database(refresh=True, force_first_time=True)

# Check mock called
rts_get_args = mock_log.getLogger().info.call_args_list
mock_log_info_calls = mock_log.getLogger().info.call_args_list
check_log1 = 'Creating database "spatial_db" for app "test_app"...'
check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...'
check_log3 = (
'Initializing database "spatial_db" for app "test_app" '
'with initializer "appsettings.model.init_spatial_db"...'
)
self.assertEqual(check_log1, rts_get_args[0][0][0])
self.assertEqual(check_log2, rts_get_args[1][0][0])
self.assertEqual(check_log3, rts_get_args[2][0][0])
self.assertEqual(check_log1, mock_log_info_calls[0][0][0])
self.assertEqual(check_log2, mock_log_info_calls[1][0][0])
self.assertEqual(check_log3, mock_log_info_calls[2][0][0])
mock_init.assert_called()

@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.drop_persistent_store_database"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.get_namespaced_persistent_store_name"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.persistent_store_database_exists"
)
@mock.patch("tethys_apps.models.PersistentStoreDatabaseSetting.get_value")
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.initializer_function"
)
@mock.patch("tethys_apps.models.logging")
def test_create_persistent_store_database_postgis2(
self, mock_log, mock_init, mock_get, mock_ps_de, mock_gn, mock_drop
):
# Mock Get Name
mock_gn.return_value = "spatial_db"

# Mock Drop Database
mock_drop.return_value = ""

# Mock persistent_store_database_exists
mock_ps_de.return_value = False # DB does not exist

# Mock get_values
mock_url = mock.MagicMock(username="test_app")
mock_engine = mock.MagicMock()
mock_new_db_engine = mock.MagicMock()
mock_db_connection = mock_new_db_engine.connect()
mock_init_param = mock.MagicMock()
mock_get.side_effect = [
mock_url,
mock_engine,
mock_new_db_engine,
mock_init_param,
]
mock_db_connection.execute.side_effect = [
mock.MagicMock(), # Enable PostGIS Statement
[
mock.MagicMock(postgis_version="2.5 USE_GEOS=1 USE_PROJ=1 USE_STATS=1")
], # Check PostGIS Version
mock.MagicMock(), # Enable PostGIS Raster Statement
]

# Execute
self.test_app.settings_set.select_subclasses().get(
name="spatial_db"
).create_persistent_store_database(refresh=False, force_first_time=False)

# Check mock calls
mock_execute_calls = mock_db_connection.execute.call_args_list
self.assertEqual(2, len(mock_execute_calls))
execute1 = "CREATE EXTENSION IF NOT EXISTS postgis;"
execute2 = "SELECT PostGIS_Version();"
self.assertEqual(execute1, mock_execute_calls[0][0][0])
self.assertEqual(execute2, mock_execute_calls[1][0][0])

mock_log_info_calls = mock_log.getLogger().info.call_args_list
self.assertEqual(4, len(mock_log_info_calls))
check_log1 = 'Creating database "spatial_db" for app "test_app"...'
check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...'
check_log3 = "Detected PostGIS version 2.5"
check_log4 = (
'Initializing database "spatial_db" for app "test_app" '
'with initializer "appsettings.model.init_spatial_db"...'
)
self.assertEqual(check_log1, mock_log_info_calls[0][0][0])
self.assertEqual(check_log2, mock_log_info_calls[1][0][0])
self.assertEqual(check_log3, mock_log_info_calls[2][0][0])
self.assertEqual(check_log4, mock_log_info_calls[3][0][0])
mock_init.assert_called()

@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.drop_persistent_store_database"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.get_namespaced_persistent_store_name"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.persistent_store_database_exists"
)
@mock.patch("tethys_apps.models.PersistentStoreDatabaseSetting.get_value")
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.initializer_function"
)
@mock.patch("tethys_apps.models.logging")
def test_create_persistent_store_database_postgis3(
self, mock_log, mock_init, mock_get, mock_ps_de, mock_gn, mock_drop
):
# Mock Get Name
mock_gn.return_value = "spatial_db"

# Mock Drop Database
mock_drop.return_value = ""

# Mock persistent_store_database_exists
mock_ps_de.return_value = False # DB does not exist

# Mock get_values
mock_url = mock.MagicMock(username="test_app")
mock_engine = mock.MagicMock()
mock_new_db_engine = mock.MagicMock()
mock_db_connection = mock_new_db_engine.connect()
mock_init_param = mock.MagicMock()
mock_get.side_effect = [
mock_url,
mock_engine,
mock_new_db_engine,
mock_init_param,
]
mock_db_connection.execute.side_effect = [
mock.MagicMock(), # Enable PostGIS Statement
[
mock.MagicMock(postgis_version="3.5 USE_GEOS=1 USE_PROJ=1 USE_STATS=1")
], # Check PostGIS Version
mock.MagicMock(), # Enable PostGIS Raster Statement
]

# Execute
self.test_app.settings_set.select_subclasses().get(
name="spatial_db"
).create_persistent_store_database(refresh=False, force_first_time=False)

# Check mock calls
mock_execute_calls = mock_db_connection.execute.call_args_list
self.assertEqual(3, len(mock_execute_calls))
execute1 = "CREATE EXTENSION IF NOT EXISTS postgis;"
execute2 = "SELECT PostGIS_Version();"
execute3 = "CREATE EXTENSION IF NOT EXISTS postgis_raster;"
self.assertEqual(execute1, mock_execute_calls[0][0][0])
self.assertEqual(execute2, mock_execute_calls[1][0][0])
self.assertEqual(execute3, mock_execute_calls[2][0][0])

mock_log_info_calls = mock_log.getLogger().info.call_args_list
self.assertEqual(5, len(mock_log_info_calls))
check_log1 = 'Creating database "spatial_db" for app "test_app"...'
check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...'
check_log3 = "Detected PostGIS version 3.5"
check_log4 = (
'Enabling PostGIS Raster on database "spatial_db" for app "test_app"...'
)
check_log5 = (
'Initializing database "spatial_db" for app "test_app" '
'with initializer "appsettings.model.init_spatial_db"...'
)
self.assertEqual(check_log1, mock_log_info_calls[0][0][0])
self.assertEqual(check_log2, mock_log_info_calls[1][0][0])
self.assertEqual(check_log3, mock_log_info_calls[2][0][0])
self.assertEqual(check_log4, mock_log_info_calls[3][0][0])
self.assertEqual(check_log5, mock_log_info_calls[4][0][0])
mock_init.assert_called()

@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.drop_persistent_store_database"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.get_namespaced_persistent_store_name"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.persistent_store_database_exists"
)
@mock.patch("tethys_apps.models.PersistentStoreDatabaseSetting.get_value")
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.initializer_function"
)
@mock.patch("tethys_apps.models.logging")
def test_create_persistent_store_database_postgis3_bad_version_string(
self, mock_log, mock_init, mock_get, mock_ps_de, mock_gn, mock_drop
):
# Mock Get Name
mock_gn.return_value = "spatial_db"

# Mock Drop Database
mock_drop.return_value = ""

# Mock persistent_store_database_exists
mock_ps_de.return_value = False # DB does not exist

# Mock get_values
mock_url = mock.MagicMock(username="test_app")
mock_engine = mock.MagicMock()
mock_new_db_engine = mock.MagicMock()
mock_db_connection = mock_new_db_engine.connect()
mock_init_param = mock.MagicMock()
mock_get.side_effect = [
mock_url,
mock_engine,
mock_new_db_engine,
mock_init_param,
]
mock_db_connection.execute.side_effect = [
mock.MagicMock(), # Enable PostGIS Statement
[
mock.MagicMock(postgis_version="BAD VERSION STRING")
], # Check PostGIS Version
mock.MagicMock(), # Enable PostGIS Raster Statement
]

# Execute
self.test_app.settings_set.select_subclasses().get(
name="spatial_db"
).create_persistent_store_database(refresh=False, force_first_time=False)

# Check mock calls
mock_execute_calls = mock_db_connection.execute.call_args_list
self.assertEqual(2, len(mock_execute_calls))
execute1 = "CREATE EXTENSION IF NOT EXISTS postgis;"
execute2 = "SELECT PostGIS_Version();"
self.assertEqual(execute1, mock_execute_calls[0][0][0])
self.assertEqual(execute2, mock_execute_calls[1][0][0])

mock_log_warning_calls = mock_log.getLogger().warning.call_args_list
self.assertEqual(1, len(mock_log_warning_calls))
check_log1 = 'Could not parse PostGIS version from "BAD VERSION STRING"'
self.assertEqual(check_log1, mock_log_warning_calls[0][0][0])

mock_log_info_calls = mock_log.getLogger().info.call_args_list
self.assertEqual(3, len(mock_log_info_calls))
check_log1 = 'Creating database "spatial_db" for app "test_app"...'
check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...'
check_log3 = (
'Initializing database "spatial_db" for app "test_app" '
'with initializer "appsettings.model.init_spatial_db"...'
)
self.assertEqual(check_log1, mock_log_info_calls[0][0][0])
self.assertEqual(check_log2, mock_log_info_calls[1][0][0])
self.assertEqual(check_log3, mock_log_info_calls[2][0][0])
mock_init.assert_called()

@mock.patch("sqlalchemy.exc")
Expand Down
37 changes: 32 additions & 5 deletions tethys_apps/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1155,19 +1155,46 @@ def create_persistent_store_database(self, refresh=False, force_first_time=False
)
)

enable_postgis_statement = "CREATE EXTENSION IF NOT EXISTS postgis"

# Execute postgis statement
try:
new_db_connection.execute(enable_postgis_statement)
new_db_connection.execute("CREATE EXTENSION IF NOT EXISTS postgis;")

# Get the POSTGIS version
ret = new_db_connection.execute("SELECT PostGIS_Version();")
postgis_version = None
for r in ret:
# Example version string: "3.4 USE_GEOS=1 USE_PROJ=1 USE_STATS=1"
try:
postgis_version = float(r.postgis_version.split(" ")[0])
log.info(f"Detected PostGIS version {postgis_version}")
break
except Exception:
log.warning(
f'Could not parse PostGIS version from "{r.postgis_version}"'
)
continue

# Execute postgis raster statement for verions 3.0 and above
if postgis_version is not None and postgis_version >= 3.0:
log.info(
'Enabling PostGIS Raster on database "{0}" for app "{1}"...'.format(
self.name,
self.tethys_app.package,
)
)
new_db_connection.execute(
"CREATE EXTENSION IF NOT EXISTS postgis_raster;"
)

except sqlalchemy.exc.ProgrammingError:
raise PersistentStorePermissionError(
'Database user "{0}" has insufficient permissions to enable '
'spatial extension on persistent store database "{1}": must be a '
"superuser.".format(url.username, self.name)
)
finally:
new_db_connection.close()

# Close connection
new_db_connection.close()

# -------------------------------------------------------------------------------------------------------------#
# 4. Run initialization function
Expand Down

0 comments on commit 25dc4a3

Please sign in to comment.