Adds beta release support for docker images (#217)

packaging_automation/common_tool_methods.py

@@ -43,6 +43,7 @@
     "ubuntu": ["focal", "bionic", "trusty"]
 }
 
+
 def platform_names() -> List[str]:
     platforms = []
     for platform_os, platform_releases in supported_platforms.items():
@@ -85,6 +86,18 @@ def get_minor_project_version(project_version: str) -> str:
     return f'{project_version_details["major"]}.{project_version_details["minor"]}'
 
 
+def project_version_contains_release_stage(project_version: str) -> bool:
+    return "_" in project_version
+
+
+def get_minor_project_version_for_docker(project_version: str) -> str:
+    project_version_details = get_version_details(project_version)
+    minor_version = f'{project_version_details["major"]}.{project_version_details["minor"]}'
+    if project_version_contains_release_stage(project_version):
+        return f'{project_version_details["stage"]}-{minor_version}'
+    return minor_version
+
+
 def append_fancy_suffix_to_version(version: str, fancy_release_number: int) -> str:
     fancy_suffix = f"-{fancy_release_number}"
     return f"{version}{fancy_suffix}"
@@ -171,7 +184,12 @@ def is_merge_commit(commit: Commit):
 @validate_parameters
 def get_version_details(version: is_version(str)) -> Dict[str, str]:
     version_parts = version.split(".")
-    return {"major": version_parts[0], "minor": version_parts[1], "patch": version_parts[2]}
+    release_stage = 'stable'
+    assert len(version_parts) == 3
+    if project_version_contains_release_stage(version):
+        stage_parts = version_parts[2].split('_')
+        release_stage = stage_parts[1]
+    return {"major": version_parts[0], "minor": version_parts[1], "patch": version_parts[2], "stage": release_stage}
 
 
 @validate_parameters
@@ -376,22 +394,28 @@ def remove_cloned_code(exec_path: str):
                   f"Please delete them manually or they will be deleted before next execution")
 
 
-def process_template_file(project_version: str, templates_path: str, template_file_path: str,
-                          postgres_version: str = ""):
+def process_template_file_with_minor(project_version: str, templates_path: str, template_file_path: str,
+                                     minor_version: str,postgres_version: str = ""):
     ''' This function gets the template files, changes tha parameters inside the file and returns the output.
         Template files are stored under packaging_automation/templates and these files include parametric items in the
         format of {{parameter_name}}. This function is used while creating docker files and pgxn files which include
         "project_name" as parameter. Example usage is in "test_common_tool_methods/test_process_template_file".
         Jinja2 is used as th the template engine and render function gets the file change parameters in the file
          with the given input parameters and returns the output.'''
-    minor_version = get_minor_project_version(project_version)
     env = get_template_environment(templates_path)
     template = env.get_template(template_file_path)
     rendered_output = template.render(project_version=project_version, postgres_version=postgres_version,
                                       project_minor_version=minor_version)
     return f"{rendered_output}\n"
 
 
+def process_template_file(project_version: str, templates_path: str, template_file_path: str,
+                          postgres_version: str = ""):
+    minor_version = get_minor_project_version(project_version)
+    return process_template_file_with_minor(project_version, templates_path, template_file_path, minor_version,
+                                            postgres_version)
+
+
 def write_to_file(content: str, dest_file_name: str):
     with open(dest_file_name, "w+", encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING,
               errors=DEFAULT_UNICODE_ERROR_HANDLER) as writer:

packaging_automation/templates/docker/alpine/alpine.tmpl.dockerfile

@@ -15,6 +15,7 @@ LABEL maintainer="Citus Data https://citusdata.com" \
 RUN apk add --no-cache \
             --virtual builddeps \
         build-base \
+        krb5-dev \
         curl \
         curl-dev \
         openssl-dev \

packaging_automation/tests/files/verify/expected_alpine_10.0.3.txt

@@ -15,6 +15,7 @@ LABEL maintainer="Citus Data https://citusdata.com" \
 RUN apk add --no-cache \
             --virtual builddeps \
         build-base \
+        krb5-dev \
         curl \
         curl-dev \
         openssl-dev \

packaging_automation/tests/test_common_tool_methods.py

@@ -79,7 +79,7 @@ def test_remove_paranthesis_from_string():
 
 
 def test_get_version_details():
-    assert get_version_details("10.0.1") == {"major": "10", "minor": "0", "patch": "1"}
+    assert get_version_details("10.0.1") == {"major": "10", "minor": "0", "patch": "1","stage":"stable"}
 
 
 def test_is_tag_on_branch():

packaging_automation/tests/test_update_docker.py

@@ -66,7 +66,7 @@ def test_update_docker_file_alpine():
         lines = content.splitlines()
         assert lines[2].strip() == f"FROM postgres:{POSTGRES_14_VERSION}-alpine"
         assert lines[3].strip() == f"ARG VERSION={PROJECT_VERSION}"
-        assert len(lines) == 55
+        assert len(lines) == 56
 
 
 def test_update_docker_file_for_postgres12():

packaging_automation/update_docker.py

@@ -8,8 +8,9 @@
 
 import pathlib2
 
-from .common_tool_methods import (process_template_file, write_to_file, run, initialize_env, create_pr,
-                                  remove_cloned_code, DEFAULT_ENCODING_FOR_FILE_HANDLING, DEFAULT_UNICODE_ERROR_HANDLER)
+from .common_tool_methods import (process_template_file_with_minor, write_to_file, run, initialize_env, create_pr,
+                                  remove_cloned_code, DEFAULT_ENCODING_FOR_FILE_HANDLING, DEFAULT_UNICODE_ERROR_HANDLER,
+                                  get_minor_project_version_for_docker)
 
 REPO_OWNER = "citusdata"
 PROJECT_NAME = "docker"
@@ -41,37 +42,49 @@ class SupportedDockerImages(Enum):
 
 def update_docker_file_for_latest_postgres(project_version: str, template_path: str, exec_path: str,
                                            postgres_version: str):
-    content = process_template_file(project_version, template_path,
-                                    docker_templates[SupportedDockerImages.latest], postgres_version)
+    minor_version = get_minor_project_version_for_docker(project_version)
+    debian_project_version = project_version.replace("_", "-")
+    content = process_template_file_with_minor(debian_project_version, template_path,
+                                               docker_templates[SupportedDockerImages.latest], minor_version,
+                                               postgres_version)
     dest_file_name = f"{exec_path}/{docker_outputs[SupportedDockerImages.latest]}"
     write_to_file(content, dest_file_name)
 
 
 def update_regular_docker_compose_file(project_version: str, template_path: str, exec_path: str):
-    content = process_template_file(project_version, template_path,
-                                    docker_templates[SupportedDockerImages.docker_compose])
+    minor_version = get_minor_project_version_for_docker(project_version)
+    content = process_template_file_with_minor(project_version, template_path,
+                                               docker_templates[SupportedDockerImages.docker_compose], minor_version)
     dest_file_name = f"{exec_path}/{docker_outputs[SupportedDockerImages.docker_compose]}"
     write_to_file(content, dest_file_name)
 
 
 def update_docker_file_alpine(project_version: str, template_path: str, exec_path: str, postgres_version: str):
-    content = process_template_file(project_version, template_path,
-                                    docker_templates[SupportedDockerImages.alpine], postgres_version)
+    minor_version = get_minor_project_version_for_docker(project_version)
+    content = process_template_file_with_minor(project_version, template_path,
+                                               docker_templates[SupportedDockerImages.alpine], minor_version,
+                                               postgres_version)
     dest_file_name = f"{exec_path}/{docker_outputs[SupportedDockerImages.alpine]}"
     write_to_file(content, dest_file_name)
 
 
 def update_docker_file_for_postgres12(project_version: str, template_path: str, exec_path: str, postgres_version: str):
-    content = process_template_file(project_version, template_path,
-                                    docker_templates[SupportedDockerImages.postgres12], postgres_version)
+    minor_version = get_minor_project_version_for_docker(project_version)
+    debian_project_version = project_version.replace("_", "-")
+    content = process_template_file_with_minor(debian_project_version, template_path,
+                                               docker_templates[SupportedDockerImages.postgres12], minor_version,
+                                               postgres_version)
     dest_file_name = f"{exec_path}/{docker_outputs[SupportedDockerImages.postgres12]}"
     create_directory_if_not_exists(dest_file_name)
     write_to_file(content, dest_file_name)
 
 
 def update_docker_file_for_postgres13(project_version: str, template_path: str, exec_path: str, postgres_version: str):
-    content = process_template_file(project_version, template_path,
-                                    docker_templates[SupportedDockerImages.postgres13], postgres_version)
+    minor_version = get_minor_project_version_for_docker(project_version)
+    debian_project_version = project_version.replace("_", "-")
+    content = process_template_file_with_minor(debian_project_version, template_path,
+                                               docker_templates[SupportedDockerImages.postgres13], minor_version,
+                                               postgres_version)
     dest_file_name = f"{exec_path}/{docker_outputs[SupportedDockerImages.postgres13]}"
     create_directory_if_not_exists(dest_file_name)
     write_to_file(content, dest_file_name)
@@ -134,7 +147,6 @@ def read_postgres_versions(pkgvars_file: str) -> Tuple[str, str, str]:
     return "14.1", "13.5", "12.9"
 
 
-
 CHECKOUT_DIR = "docker_temp"
 
 if __name__ == "__main__":