diff --git a/.github/workflows/tethys-release.yml b/.github/workflows/tethys-release.yml index 0a7cff745..0968e3738 100644 --- a/.github/workflows/tethys-release.yml +++ b/.github/workflows/tethys-release.yml @@ -9,6 +9,7 @@ on: env: CONDA_BUILD_PIN_LEVEL: minor + DOCKER_UPLOAD_URL: tethysplatform/tethys-core jobs: docker-build: @@ -58,7 +59,7 @@ jobs: full_tag="${{ steps.version.outputs.full }}"; # no "+" characters allowed in Docker tags safe_tag="${full_tag//+/-}"; - docker build -t ${{ secrets.DOCKER_UPLOAD_URL }}:"${{ steps.safetag.outputs.safetag }}" .; + docker build -t ${{env.DOCKER_UPLOAD_URL }}:"${{ steps.safetag.outputs.safetag }}" .; # Authenticate docker - name: Authenticate Docker run: | @@ -67,13 +68,13 @@ jobs: - name: Upload Docker With Tag run: | echo "Pushing to docker registry"; - docker push ${{ secrets.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }}; + docker push ${{ env.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }}; - name: Upload Docker With Latest Tag if: ${{ steps.version.outputs.prerelease == '' }} run: | echo "Updating latest on the docker registry"; - docker tag ${{ secrets.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }} ${{ secrets.DOCKER_UPLOAD_URL }}:latest; - docker push ${{ secrets.DOCKER_UPLOAD_URL }}:latest; + docker tag ${{ env.DOCKER_UPLOAD_URL }}:${{ steps.safetag.outputs.safetag }} ${{env.DOCKER_UPLOAD_URL }}:latest; + docker push ${{ env.DOCKER_UPLOAD_URL }}:latest; conda-build: name: Conda Build (${{ matrix.platform }}) diff --git a/.github/workflows/tethys.yml b/.github/workflows/tethys.yml index e91e8bc45..7f3737e5e 100644 --- a/.github/workflows/tethys.yml +++ b/.github/workflows/tethys.yml @@ -15,6 +15,7 @@ on: env: CONDA_BUILD_PIN_LEVEL: minor + DOCKER_UPLOAD_URL: tethysplatform/tethys-core GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TEST_IMAGE: tethys:dev POSTGRES_DB: tethys_postgis @@ -107,15 +108,15 @@ jobs: # Build the docker for no tag - name: Build Without Tag run: | - docker build -t ${{ secrets.DOCKER_UPLOAD_URL }}:dev .; - docker tag ${{ secrets.DOCKER_UPLOAD_URL }}:dev ${{ env.TEST_IMAGE }}; + docker build -t ${{ env.DOCKER_UPLOAD_URL }}:4.2-dev .; + docker tag ${{ env.DOCKER_UPLOAD_URL }}:4.2-dev ${{ env.TEST_IMAGE }}; # Upload docker if pull request no tag - name: Upload Docker No Tag if: ${{ github.event_name != 'pull_request' }} run: | echo "Pushing to docker registry"; echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin; - docker push ${{ secrets.DOCKER_UPLOAD_URL }}:dev; + docker push ${{ env.DOCKER_UPLOAD_URL }}:4.2-dev; # No Upload if Pull Request - name: No Upload if: ${{ github.event_name == 'pull_request' }} diff --git a/tests/unit_tests/test_tethys_apps/test_models/test_PersistentStoreDatabaseSetting.py b/tests/unit_tests/test_tethys_apps/test_models/test_PersistentStoreDatabaseSetting.py index 9f2f4de86..1bba037d0 100644 --- a/tests/unit_tests/test_tethys_apps/test_models/test_PersistentStoreDatabaseSetting.py +++ b/tests/unit_tests/test_tethys_apps/test_models/test_PersistentStoreDatabaseSetting.py @@ -427,16 +427,247 @@ def test_create_persistent_store_database( ).create_persistent_store_database(refresh=True, force_first_time=True) # Check mock called - rts_get_args = mock_log.getLogger().info.call_args_list + mock_log_info_calls = mock_log.getLogger().info.call_args_list check_log1 = 'Creating database "spatial_db" for app "test_app"...' check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...' check_log3 = ( 'Initializing database "spatial_db" for app "test_app" ' 'with initializer "appsettings.model.init_spatial_db"...' ) - self.assertEqual(check_log1, rts_get_args[0][0][0]) - self.assertEqual(check_log2, rts_get_args[1][0][0]) - self.assertEqual(check_log3, rts_get_args[2][0][0]) + self.assertEqual(check_log1, mock_log_info_calls[0][0][0]) + self.assertEqual(check_log2, mock_log_info_calls[1][0][0]) + self.assertEqual(check_log3, mock_log_info_calls[2][0][0]) + mock_init.assert_called() + + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.drop_persistent_store_database" + ) + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.get_namespaced_persistent_store_name" + ) + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.persistent_store_database_exists" + ) + @mock.patch("tethys_apps.models.PersistentStoreDatabaseSetting.get_value") + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.initializer_function" + ) + @mock.patch("tethys_apps.models.logging") + def test_create_persistent_store_database_postgis2( + self, mock_log, mock_init, mock_get, mock_ps_de, mock_gn, mock_drop + ): + # Mock Get Name + mock_gn.return_value = "spatial_db" + + # Mock Drop Database + mock_drop.return_value = "" + + # Mock persistent_store_database_exists + mock_ps_de.return_value = False # DB does not exist + + # Mock get_values + mock_url = mock.MagicMock(username="test_app") + mock_engine = mock.MagicMock() + mock_new_db_engine = mock.MagicMock() + mock_db_connection = mock_new_db_engine.connect() + mock_init_param = mock.MagicMock() + mock_get.side_effect = [ + mock_url, + mock_engine, + mock_new_db_engine, + mock_init_param, + ] + mock_db_connection.execute.side_effect = [ + mock.MagicMock(), # Enable PostGIS Statement + [ + mock.MagicMock(postgis_version="2.5 USE_GEOS=1 USE_PROJ=1 USE_STATS=1") + ], # Check PostGIS Version + mock.MagicMock(), # Enable PostGIS Raster Statement + ] + + # Execute + self.test_app.settings_set.select_subclasses().get( + name="spatial_db" + ).create_persistent_store_database(refresh=False, force_first_time=False) + + # Check mock calls + mock_execute_calls = mock_db_connection.execute.call_args_list + self.assertEqual(2, len(mock_execute_calls)) + execute1 = "CREATE EXTENSION IF NOT EXISTS postgis;" + execute2 = "SELECT PostGIS_Version();" + self.assertEqual(execute1, mock_execute_calls[0][0][0]) + self.assertEqual(execute2, mock_execute_calls[1][0][0]) + + mock_log_info_calls = mock_log.getLogger().info.call_args_list + self.assertEqual(4, len(mock_log_info_calls)) + check_log1 = 'Creating database "spatial_db" for app "test_app"...' + check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...' + check_log3 = "Detected PostGIS version 2.5" + check_log4 = ( + 'Initializing database "spatial_db" for app "test_app" ' + 'with initializer "appsettings.model.init_spatial_db"...' + ) + self.assertEqual(check_log1, mock_log_info_calls[0][0][0]) + self.assertEqual(check_log2, mock_log_info_calls[1][0][0]) + self.assertEqual(check_log3, mock_log_info_calls[2][0][0]) + self.assertEqual(check_log4, mock_log_info_calls[3][0][0]) + mock_init.assert_called() + + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.drop_persistent_store_database" + ) + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.get_namespaced_persistent_store_name" + ) + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.persistent_store_database_exists" + ) + @mock.patch("tethys_apps.models.PersistentStoreDatabaseSetting.get_value") + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.initializer_function" + ) + @mock.patch("tethys_apps.models.logging") + def test_create_persistent_store_database_postgis3( + self, mock_log, mock_init, mock_get, mock_ps_de, mock_gn, mock_drop + ): + # Mock Get Name + mock_gn.return_value = "spatial_db" + + # Mock Drop Database + mock_drop.return_value = "" + + # Mock persistent_store_database_exists + mock_ps_de.return_value = False # DB does not exist + + # Mock get_values + mock_url = mock.MagicMock(username="test_app") + mock_engine = mock.MagicMock() + mock_new_db_engine = mock.MagicMock() + mock_db_connection = mock_new_db_engine.connect() + mock_init_param = mock.MagicMock() + mock_get.side_effect = [ + mock_url, + mock_engine, + mock_new_db_engine, + mock_init_param, + ] + mock_db_connection.execute.side_effect = [ + mock.MagicMock(), # Enable PostGIS Statement + [ + mock.MagicMock(postgis_version="3.5 USE_GEOS=1 USE_PROJ=1 USE_STATS=1") + ], # Check PostGIS Version + mock.MagicMock(), # Enable PostGIS Raster Statement + ] + + # Execute + self.test_app.settings_set.select_subclasses().get( + name="spatial_db" + ).create_persistent_store_database(refresh=False, force_first_time=False) + + # Check mock calls + mock_execute_calls = mock_db_connection.execute.call_args_list + self.assertEqual(3, len(mock_execute_calls)) + execute1 = "CREATE EXTENSION IF NOT EXISTS postgis;" + execute2 = "SELECT PostGIS_Version();" + execute3 = "CREATE EXTENSION IF NOT EXISTS postgis_raster;" + self.assertEqual(execute1, mock_execute_calls[0][0][0]) + self.assertEqual(execute2, mock_execute_calls[1][0][0]) + self.assertEqual(execute3, mock_execute_calls[2][0][0]) + + mock_log_info_calls = mock_log.getLogger().info.call_args_list + self.assertEqual(5, len(mock_log_info_calls)) + check_log1 = 'Creating database "spatial_db" for app "test_app"...' + check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...' + check_log3 = "Detected PostGIS version 3.5" + check_log4 = ( + 'Enabling PostGIS Raster on database "spatial_db" for app "test_app"...' + ) + check_log5 = ( + 'Initializing database "spatial_db" for app "test_app" ' + 'with initializer "appsettings.model.init_spatial_db"...' + ) + self.assertEqual(check_log1, mock_log_info_calls[0][0][0]) + self.assertEqual(check_log2, mock_log_info_calls[1][0][0]) + self.assertEqual(check_log3, mock_log_info_calls[2][0][0]) + self.assertEqual(check_log4, mock_log_info_calls[3][0][0]) + self.assertEqual(check_log5, mock_log_info_calls[4][0][0]) + mock_init.assert_called() + + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.drop_persistent_store_database" + ) + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.get_namespaced_persistent_store_name" + ) + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.persistent_store_database_exists" + ) + @mock.patch("tethys_apps.models.PersistentStoreDatabaseSetting.get_value") + @mock.patch( + "tethys_apps.models.PersistentStoreDatabaseSetting.initializer_function" + ) + @mock.patch("tethys_apps.models.logging") + def test_create_persistent_store_database_postgis3_bad_version_string( + self, mock_log, mock_init, mock_get, mock_ps_de, mock_gn, mock_drop + ): + # Mock Get Name + mock_gn.return_value = "spatial_db" + + # Mock Drop Database + mock_drop.return_value = "" + + # Mock persistent_store_database_exists + mock_ps_de.return_value = False # DB does not exist + + # Mock get_values + mock_url = mock.MagicMock(username="test_app") + mock_engine = mock.MagicMock() + mock_new_db_engine = mock.MagicMock() + mock_db_connection = mock_new_db_engine.connect() + mock_init_param = mock.MagicMock() + mock_get.side_effect = [ + mock_url, + mock_engine, + mock_new_db_engine, + mock_init_param, + ] + mock_db_connection.execute.side_effect = [ + mock.MagicMock(), # Enable PostGIS Statement + [ + mock.MagicMock(postgis_version="BAD VERSION STRING") + ], # Check PostGIS Version + mock.MagicMock(), # Enable PostGIS Raster Statement + ] + + # Execute + self.test_app.settings_set.select_subclasses().get( + name="spatial_db" + ).create_persistent_store_database(refresh=False, force_first_time=False) + + # Check mock calls + mock_execute_calls = mock_db_connection.execute.call_args_list + self.assertEqual(2, len(mock_execute_calls)) + execute1 = "CREATE EXTENSION IF NOT EXISTS postgis;" + execute2 = "SELECT PostGIS_Version();" + self.assertEqual(execute1, mock_execute_calls[0][0][0]) + self.assertEqual(execute2, mock_execute_calls[1][0][0]) + + mock_log_warning_calls = mock_log.getLogger().warning.call_args_list + self.assertEqual(1, len(mock_log_warning_calls)) + check_log1 = 'Could not parse PostGIS version from "BAD VERSION STRING"' + self.assertEqual(check_log1, mock_log_warning_calls[0][0][0]) + + mock_log_info_calls = mock_log.getLogger().info.call_args_list + self.assertEqual(3, len(mock_log_info_calls)) + check_log1 = 'Creating database "spatial_db" for app "test_app"...' + check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...' + check_log3 = ( + 'Initializing database "spatial_db" for app "test_app" ' + 'with initializer "appsettings.model.init_spatial_db"...' + ) + self.assertEqual(check_log1, mock_log_info_calls[0][0][0]) + self.assertEqual(check_log2, mock_log_info_calls[1][0][0]) + self.assertEqual(check_log3, mock_log_info_calls[2][0][0]) mock_init.assert_called() @mock.patch("sqlalchemy.exc") diff --git a/tethys_apps/models.py b/tethys_apps/models.py index b39e36644..e95589420 100644 --- a/tethys_apps/models.py +++ b/tethys_apps/models.py @@ -1155,19 +1155,46 @@ def create_persistent_store_database(self, refresh=False, force_first_time=False ) ) - enable_postgis_statement = "CREATE EXTENSION IF NOT EXISTS postgis" - # Execute postgis statement try: - new_db_connection.execute(enable_postgis_statement) + new_db_connection.execute("CREATE EXTENSION IF NOT EXISTS postgis;") + + # Get the POSTGIS version + ret = new_db_connection.execute("SELECT PostGIS_Version();") + postgis_version = None + for r in ret: + # Example version string: "3.4 USE_GEOS=1 USE_PROJ=1 USE_STATS=1" + try: + postgis_version = float(r.postgis_version.split(" ")[0]) + log.info(f"Detected PostGIS version {postgis_version}") + break + except Exception: + log.warning( + f'Could not parse PostGIS version from "{r.postgis_version}"' + ) + continue + + # Execute postgis raster statement for verions 3.0 and above + if postgis_version is not None and postgis_version >= 3.0: + log.info( + 'Enabling PostGIS Raster on database "{0}" for app "{1}"...'.format( + self.name, + self.tethys_app.package, + ) + ) + new_db_connection.execute( + "CREATE EXTENSION IF NOT EXISTS postgis_raster;" + ) + except sqlalchemy.exc.ProgrammingError: raise PersistentStorePermissionError( 'Database user "{0}" has insufficient permissions to enable ' 'spatial extension on persistent store database "{1}": must be a ' "superuser.".format(url.username, self.name) ) - finally: - new_db_connection.close() + + # Close connection + new_db_connection.close() # -------------------------------------------------------------------------------------------------------------# # 4. Run initialization function