From aaf859053c7a6200527e0ae1010a7bbc10727ccf Mon Sep 17 00:00:00 2001 From: panbingkun Date: Fri, 29 Nov 2024 09:15:37 +0800 Subject: [PATCH] [SPARK-50448][SQL][TESTS] Extract postgres image common conf as PostgresDatabaseOnDocker ### What changes were proposed in this pull request? The pr aims to extract `postgres` image `common` conf as `PostgresDatabaseOnDocker`. ### Why are the changes needed? Reduce the cost of the next `postgres` image upgrade. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass existed UT & GA. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #49004 from panbingkun/SPARK-50448. Authored-by: panbingkun Signed-off-by: panbingkun --- .../sql/jdbc/PostgresDatabaseOnDocker.scala | 36 +++++++++++++++++++ .../sql/jdbc/PostgresIntegrationSuite.scala | 11 +----- .../jdbc/PostgresKrbIntegrationSuite.scala | 9 +---- .../querytest/GeneratedSubquerySuite.scala | 11 +----- ....scala => PostgresSQLQueryTestSuite.scala} | 14 ++------ .../jdbc/v2/PostgresIntegrationSuite.scala | 13 ++----- .../sql/jdbc/v2/PostgresNamespaceSuite.scala | 13 ++----- 7 files changed, 45 insertions(+), 62 deletions(-) create mode 100644 connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresDatabaseOnDocker.scala rename connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/{PostgreSQLQueryTestSuite.scala => PostgresSQLQueryTestSuite.scala} (83%) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresDatabaseOnDocker.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresDatabaseOnDocker.scala new file mode 100644 index 0000000000000..db2495ad3c698 --- /dev/null +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresDatabaseOnDocker.scala @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.jdbc + +import org.apache.spark.internal.Logging + +class PostgresDatabaseOnDocker extends DatabaseOnDocker with Logging { + lazy override val imageName: String = + sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:17.2-alpine") + private val postgres_user = "postgres" + private val postgres_password = "rootpass" + override val env: Map[String, String] = Map( + "POSTGRES_PASSWORD" -> postgres_password + ) + override val usesIpc = false + override val jdbcPort: Int = 5432 + + override def getJdbcUrl(ip: String, port: Int): String = { + s"jdbc:postgresql://$ip:$port/postgres?user=$postgres_user&password=$postgres_password" + } +} diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala index c3a705a1c7e28..5c985da226b06 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala @@ -41,16 +41,7 @@ import org.apache.spark.tags.DockerTest */ @DockerTest class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite { - override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:17.2-alpine") - override val env = Map( - "POSTGRES_PASSWORD" -> "rootpass" - ) - override val usesIpc = false - override val jdbcPort = 5432 - override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:postgresql://$ip:$port/postgres?user=postgres&password=rootpass" - } + override val db = new PostgresDatabaseOnDocker override def dataPreparation(conn: Connection): Unit = { conn.prepareStatement("CREATE DATABASE foo").executeUpdate() diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresKrbIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresKrbIntegrationSuite.scala index 6bb6bbc36a2d4..b3cfe8bd77e2b 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresKrbIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresKrbIntegrationSuite.scala @@ -37,14 +37,7 @@ class PostgresKrbIntegrationSuite extends DockerKrbJDBCIntegrationSuite { override protected val userName = s"postgres/$dockerIp" override protected val keytabFileName = "postgres.keytab" - override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:17.2-alpine") - override val env = Map( - "POSTGRES_PASSWORD" -> "rootpass" - ) - override val usesIpc = false - override val jdbcPort = 5432 - + override val db = new PostgresDatabaseOnDocker { override def getJdbcUrl(ip: String, port: Int): String = s"jdbc:postgresql://$ip:$port/postgres?user=$principal&gsslib=gssapi" diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala index c51ae4c9ed71a..3a1d5e18b7e5a 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/GeneratedSubquerySuite.scala @@ -38,16 +38,7 @@ import org.apache.spark.tags.DockerTest @DockerTest class GeneratedSubquerySuite extends DockerJDBCIntegrationSuite with QueryGeneratorHelper { - override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:17.2-alpine") - override val env = Map( - "POSTGRES_PASSWORD" -> "rootpass" - ) - override val usesIpc = false - override val jdbcPort = 5432 - override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:postgresql://$ip:$port/postgres?user=postgres&password=rootpass" - } + override val db = new PostgresDatabaseOnDocker private val FIRST_COLUMN = "a" private val SECOND_COLUMN = "b" diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgreSQLQueryTestSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala similarity index 83% rename from connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgreSQLQueryTestSuite.scala rename to connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala index 84a04cafb6453..28320a9e0a949 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgreSQLQueryTestSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/PostgresSQLQueryTestSuite.scala @@ -38,23 +38,13 @@ import org.apache.spark.tags.DockerTest * }}} */ @DockerTest -class PostgreSQLQueryTestSuite extends CrossDbmsQueryTestSuite { +class PostgresSQLQueryTestSuite extends CrossDbmsQueryTestSuite { val DATABASE_NAME = CrossDbmsQueryTestSuite.POSTGRES // Scope to only subquery directory for now. protected val customInputFilePath: String = new File(inputFilePath, "subquery").getAbsolutePath - override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:17.2-alpine") - override val env = Map( - "POSTGRES_PASSWORD" -> "rootpass" - ) - override val usesIpc = false - override val jdbcPort = 5432 - - override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:postgresql://$ip:$port/postgres?user=postgres&password=rootpass" - } + override val db = new PostgresDatabaseOnDocker override def dataPreparation(conn: Connection): Unit = { conn.prepareStatement( diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresIntegrationSuite.scala index ce79618a011cb..ef52aebd723a6 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresIntegrationSuite.scala @@ -23,7 +23,7 @@ import org.apache.spark.{SparkConf, SparkSQLException} import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog -import org.apache.spark.sql.jdbc.DatabaseOnDocker +import org.apache.spark.sql.jdbc.PostgresDatabaseOnDocker import org.apache.spark.sql.types._ import org.apache.spark.tags.DockerTest @@ -37,16 +37,7 @@ import org.apache.spark.tags.DockerTest @DockerTest class PostgresIntegrationSuite extends DockerJDBCIntegrationV2Suite with V2JDBCTest { override val catalogName: String = "postgresql" - override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:17.2-alpine") - override val env = Map( - "POSTGRES_PASSWORD" -> "rootpass" - ) - override val usesIpc = false - override val jdbcPort = 5432 - override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:postgresql://$ip:$port/postgres?user=postgres&password=rootpass" - } + override val db = new PostgresDatabaseOnDocker override def sparkConf: SparkConf = super.sparkConf .set("spark.sql.catalog.postgresql", classOf[JDBCTableCatalog].getName) .set("spark.sql.catalog.postgresql.url", db.getJdbcUrl(dockerIp, externalPort)) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresNamespaceSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresNamespaceSuite.scala index 649267788ad60..f84bdb46850f2 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresNamespaceSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresNamespaceSuite.scala @@ -21,7 +21,7 @@ import java.sql.Connection import scala.jdk.CollectionConverters._ -import org.apache.spark.sql.jdbc.{DatabaseOnDocker, DockerJDBCIntegrationSuite} +import org.apache.spark.sql.jdbc.{DockerJDBCIntegrationSuite, PostgresDatabaseOnDocker} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.tags.DockerTest @@ -34,16 +34,7 @@ import org.apache.spark.tags.DockerTest */ @DockerTest class PostgresNamespaceSuite extends DockerJDBCIntegrationSuite with V2JDBCNamespaceTest { - override val db = new DatabaseOnDocker { - override val imageName = sys.env.getOrElse("POSTGRES_DOCKER_IMAGE_NAME", "postgres:17.2-alpine") - override val env = Map( - "POSTGRES_PASSWORD" -> "rootpass" - ) - override val usesIpc = false - override val jdbcPort = 5432 - override def getJdbcUrl(ip: String, port: Int): String = - s"jdbc:postgresql://$ip:$port/postgres?user=postgres&password=rootpass" - } + override val db = new PostgresDatabaseOnDocker val map = new CaseInsensitiveStringMap( Map("url" -> db.getJdbcUrl(dockerIp, externalPort),