feat(env variable): add write of environnement variable in .ini files…

… from QDT profile (#452)

- add write of environnement variable in .ini files with implementation
of `before_write` in `EnvironmentVariablesInterpolation`
- add merge of `QdtProfile` from installed QGIS folder and downloaded
one:
    - copy download folder to temporary folder
    - copy available .ini files from installed QGIS folder
    - merge .ini from download folder and backup updated values

closes #451

.github/workflows/tests.yml

@@ -3,14 +3,14 @@ name: "🎳 Tester"
 on:
   push:
     branches:
-        - main
+      - main
     paths:
       - "**/*.py"
       - ".github/workflows/tests.yml"
 
   pull_request:
     branches:
-        - main
+      - main
     paths:
       - "**/*.py"
       - ".github/workflows/tests.yml"
@@ -32,9 +32,9 @@ jobs:
           - ubuntu-22.04
           - windows-latest
         python-version:
-            - "3.10"
-            - "3.11"
-            - "3.12"
+          - "3.10"
+          - "3.11"
+          - "3.12"
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -84,9 +84,9 @@ jobs:
           - ubuntu-22.04
           - windows-latest
         python-version:
-            - "3.10"
-            - "3.11"
-            - "3.12"
+          - "3.10"
+          - "3.11"
+          - "3.12"
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -114,8 +114,15 @@ jobs:
       - name: QDT - Echoing version
         run: qdeploy-toolbelt --version
 
-      - name: QDT - Sample scenario
-        run: qgis-deployment-toolbelt --verbose
+      - name: QDT - Check upgrade
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # required to avoid GH API rate limit
+        run: qdt upgrade
+
+      - name: QDT - Run sample scenario twice
+        run: |
+          qgis-deployment-toolbelt --verbose
+          qgis-deployment-toolbelt
 
-      - name: QDT - Sample scenario
+      - name: QDT - Run sample scenario a third time
         run: qgis-deployment-toolbelt --verbose --no-logfile

examples/profiles/Viewer Mode/QGIS/QGIS3.ini

@@ -1169,3 +1169,5 @@ searchPathsForSVG=/usr/share/qgis/svg/, /home/jmo/Git/Oslandia/QGIS/qgis-deploym
 
 [variables]
 created_with_qdt=true
+current_user=$USER
+updated_variable=profile_variable

examples/profiles/demo/QGIS/QGIS3.ini

@@ -164,3 +164,5 @@ searchPathsForSVG=/usr/share/qgis/svg/
 
 [variables]
 created_with_qdt=true
+current_user=$USER
+updated_variable=profile_variable

qgis_deployment_toolbelt/jobs/job_profiles_synchronizer.py

@@ -272,11 +272,16 @@ def sync_overwrite_local_profiles(
 
         # copy downloaded profiles into this
         for d in profiles_to_copy:
-            logger.info(f"Copying {d.folder} to {d.path_in_qgis}")
-            d.path_in_qgis.mkdir(parents=True, exist_ok=True)
-            copytree(
-                d.folder,
-                d.path_in_qgis,
-                copy_function=copy2,
-                dirs_exist_ok=True,
-            )
+            if d.path_in_qgis.exists():
+                logger.info(f"Merging {d.folder} to {d.path_in_qgis}")
+                installed_profile = QdtProfile(folder=d.path_in_qgis)
+                d.merge_to(installed_profile)
+            else:
+                logger.info(f"Copying {d.folder} to {d.path_in_qgis}")
+                d.path_in_qgis.mkdir(parents=True, exist_ok=True)
+                copytree(
+                    d.folder,
+                    d.path_in_qgis,
+                    copy_function=copy2,
+                    dirs_exist_ok=True,
+                )

qgis_deployment_toolbelt/profiles/qdt_profile.py

@@ -17,7 +17,9 @@
 # standard
 import json
 import logging
+import tempfile
 from pathlib import Path
+from shutil import copy2, copytree
 from typing import Literal
 
 # 3rd party
@@ -451,6 +453,62 @@ def get_qgis3customizationini_helper(self) -> QgisIniHelper:
             ini_type="profile_qgis3customization",
         )
 
+    def merge_to(self, dst: QdtProfile) -> None:
+        """Merge QdtProfile to another profile
+
+
+        Args:
+            dst (QdtProfile): _description_
+        """
+        with tempfile.TemporaryDirectory(
+            prefix=f"QDT_merge_profile_{self.name}_", ignore_cleanup_errors=True
+        ) as tmpdirname:
+            logger.info(
+                f"Merge profile {self.name} with {tmpdirname} temporary directory"
+            )
+            # Copy source QdtProfile folder
+            copytree(
+                self.folder,
+                tmpdirname,
+                copy_function=copy2,
+                dirs_exist_ok=True,
+            )
+            # Merge INI files
+            tmp_profile = QdtProfile(folder=Path(tmpdirname))
+
+            # QGIS3
+            if self.has_qgis3_ini_file() and dst.has_qgis3_ini_file():
+                # Copy current installed file
+                copy2(
+                    dst.get_qgis3ini_helper().ini_filepath,
+                    tmp_profile.get_qgis3ini_helper().ini_filepath,
+                )
+                # Merge
+                self.get_qgis3ini_helper().merge_to(tmp_profile.get_qgis3ini_helper())
+
+            # QGISCUSTOMIZATION3
+            if (
+                self.has_qgis3customization_ini_file()
+                and dst.has_qgis3customization_ini_file()
+            ):
+                # Copy current installed file
+                copy2(
+                    dst.get_qgis3customizationini_helper().ini_filepath,
+                    tmp_profile.get_qgis3customizationini_helper().ini_filepath,
+                )
+                # Merge
+                self.get_qgis3customizationini_helper().merge_to(
+                    tmp_profile.get_qgis3customizationini_helper()
+                )
+
+            logger.info(f"Copying {tmpdirname} to {dst.path_in_qgis}")
+            copytree(
+                tmpdirname,
+                dst.path_in_qgis,
+                copy_function=copy2,
+                dirs_exist_ok=True,
+            )
+
 
 # #############################################################################
 # ##### Stand alone program ########

qgis_deployment_toolbelt/profiles/qgis_ini_handler.py

@@ -11,6 +11,9 @@
 # ########## Libraries #############
 # ##################################
 
+# special
+from __future__ import annotations
+
 # Standard library
 import logging
 from pathlib import Path
@@ -112,6 +115,7 @@ def __init__(
             raise_error=False,
         ):
             logger.info(f"The specified file does not exist: {ini_filepath.resolve()}.")
+        self.ini_filepath = ini_filepath
 
         # store options
         self.strict_mode = strict
@@ -542,6 +546,93 @@ def set_splash_screen(
             )
             return False
 
+    @staticmethod
+    def _copy_section(
+        config_dest: CustomConfigParser,
+        config_src: CustomConfigParser,
+        section_src: str,
+    ) -> None:
+        """Copy section from an INI config to another INI config
+
+        Args:
+            config_dest (CustomConfigParser): destination INI content
+            config_src (CustomConfigParser): source INI content
+            section_src (str): section to copy
+        """
+        if section_src not in config_dest.sections() and section_src != "DEFAULT":
+            config_dest.add_section(section_src)
+        for param in config_src[section_src]:
+            config_dest[section_src][param] = config_src[section_src][param]
+
+    @staticmethod
+    def _backup_section(
+        config_dest: CustomConfigParser,
+        config_src: CustomConfigParser,
+        section_src: str,
+        dst: Path,
+    ) -> None:
+        """Backup a INI section for with updated values
+
+        Args:
+            config_dest (CustomConfigParser): destination INI content
+            config_src (CustomConfigParser): source INI content
+            section_src (str): section to backup
+            dst (Path): destination file
+        """
+        if section_src not in config_dest:
+            return
+        # Get updated values
+        updated_values = {
+            param: config_dest[section_src][param]
+            for param in config_src[section_src]
+            if param in config_dest[section_src]
+            and config_src[section_src][param] != config_dest[section_src][param]
+        }
+        if len(updated_values):
+            backup_section = f"QDT_backup_{section_src}"
+            logger.info(
+                f"Section {section_src} already available in {dst}. Copying updated content to {backup_section}"
+            )
+            if backup_section not in config_dest.sections():
+                config_dest.add_section(backup_section)
+            for param, backup_val in updated_values.items():
+                config_dest[backup_section][param] = backup_val
+
+    def merge_to(self, dst: QgisIniHelper) -> None:
+        """Merge INI file to another INI file.
+        If the destination file exists a merge is done:
+        - all available sections are kept
+        - if a section is available in both INI files, keep updated parameters in a backup section
+        If environnement variable interpolation is enabled, value are written with current environnement values
+
+        Args:
+            dst (QgisIniHelper): destination ini file
+        """
+        if not self.ini_filepath or not self.ini_filepath.exists():
+            logger.warning(
+                f"File {self.ini_filepath} doesn't exists. Can't merge to {dst}"
+            )
+            return
+
+        # Read source INI content
+        config_src = self.cfg_parser()
+        config_src.read(self.ini_filepath)
+
+        if dst.ini_filepath.exists():
+            # Read destination INI content
+            config_dest = dst.cfg_parser()
+            config_dest.read(dst.ini_filepath)
+            # Add sections in source INI
+            for section in config_src:
+                self._backup_section(config_dest, config_src, section, dst.ini_filepath)
+                self._copy_section(config_dest, config_src, section)
+            # Write to destination, environnement variable will be interpolated if interpolation enabled
+            with dst.ini_filepath.open("w") as config_file:
+                config_dest.write(config_file)
+        else:
+            with dst.ini_filepath.open("w") as config_file:
+                config_src.write(config_file)
+
 
 # #############################################################################
 # ##### Stand alone program ########

qgis_deployment_toolbelt/utils/ini_interpolation.py

@@ -14,7 +14,7 @@
 # Standard library
 import logging
 from collections import ChainMap
-from configparser import BasicInterpolation, ConfigParser
+from configparser import BasicInterpolation, ConfigParser, InterpolationSyntaxError
 from os.path import expanduser, expandvars
 
 # #############################################################################
@@ -54,7 +54,7 @@ def before_get(
         value: str,
         defaults: ChainMap,
     ) -> str:
-        """Called for every option=value line in INI file.
+        """Called for every get option=value line in INI file.
 
         Args:
             parser (ConfigParser): parser whose function is overloaded
@@ -66,7 +66,62 @@ def before_get(
         Returns:
             str: interpolated value
         """
-        value = super().before_get(parser, section, option, value, defaults)
+        # Add try catch because QGIS INI file can omit some % escaping
+        try:
+            value = super().before_get(parser, section, option, value, defaults)
+        except InterpolationSyntaxError:
+            return value
+
+        try:
+            return expandvars(expanduser(value))
+        except Exception as exc:
+            logger.error(
+                f"Failed to interpolate {value} in {section}/{option}. Trace: {exc}"
+            )
+            return value
+
+    def before_set(
+        self,
+        parser: ConfigParser,
+        section: str,
+        option: str,
+        value: str,
+    ) -> str:
+        """Called for every set option=value line in INI file.
+
+        Args:
+            parser (ConfigParser): parser whose function is overloaded
+            section (str): section's name
+            option (str): option's name
+            value (str): value to try to interpolate
+        Returns:
+            str: interpolated value
+        """
+        # Add try catch because QGIS INI file can omit some % escaping
+        try:
+            return super().before_set(parser, section, option, value)
+        except (InterpolationSyntaxError, ValueError):
+            return value
+
+    def before_write(
+        self,
+        parser: ConfigParser,
+        section: str,
+        option: str,
+        value: str,
+    ) -> str:
+        """Called before write option=value line in INI file.
+
+        Args:
+            parser (ConfigParser): parser whose function is overloaded
+            section (str): section's name
+            option (str): option's name
+            value (str): value to try to interpolate
+
+        Returns:
+            str: interpolated value
+        """
+        value = super().before_write(parser, section, option, value)
         try:
             return expandvars(expanduser(value))
         except Exception as exc: