From 6e421105d808127ec8f32a97d814938d1e60c6b6 Mon Sep 17 00:00:00 2001 From: jmkerloch Date: Mon, 7 Oct 2024 09:54:01 +0200 Subject: [PATCH 1/2] feat(pypac): add tldextract cache directory in pyinstaller --- .github/workflows/build_release.yml | 8 +++ builder/pyinstaller_build_macos.py | 2 + builder/pyinstaller_build_ubuntu.py | 2 + builder/pyinstaller_build_windows.py | 2 + builder/tldextract_update.py | 83 ++++++++++++++++++++++++++++ docs/development/packaging.md | 5 ++ 6 files changed, 102 insertions(+) create mode 100644 builder/tldextract_update.py diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml index c0cac18a..f7c60811 100644 --- a/.github/workflows/build_release.yml +++ b/.github/workflows/build_release.yml @@ -92,6 +92,9 @@ jobs: - name: Install project as a package run: python -m pip install -e . + - name: Generates tldextract cache + run: python -O ./builder/tldextract_update.py + - name: Generates Executable run: python -O ./builder/pyinstaller_build_macos.py @@ -126,6 +129,8 @@ jobs: - name: Install project as a package run: python -m pip install -e . + - name: Generates tldextract cache + run: python -O ./builder/tldextract_update.py - name: Generates Executable run: python -O ./builder/pyinstaller_build_ubuntu.py @@ -165,6 +170,9 @@ jobs: - name: Generates MS Version Info run: python .\builder\version_info_templater.py + - name: Generates tldextract cache + run: python -O .\builder\tldextract_update.py + - name: Generates MS Executable run: python -O .\builder\pyinstaller_build_windows.py diff --git a/builder/pyinstaller_build_macos.py b/builder/pyinstaller_build_macos.py index a075efd8..8b5099a5 100644 --- a/builder/pyinstaller_build_macos.py +++ b/builder/pyinstaller_build_macos.py @@ -53,6 +53,8 @@ [ "--add-data=LICENSE:.", "--add-data=README.md:.", + f"--add-data={Path(__file__).parent / 'build' / 'tldextract_cache'/ '.suffix_cache'}:tldextract/.suffix_cache", + f"--add-data={Path(__file__).parent / 'build' / 'tldextract_cache'/'.tld_set_snapshot'}:tldextract/", f"--add-data={package_folder.joinpath('shortcuts/shortcut_freedesktop.template').resolve()}:profiles/", f"--log-level={getenv('PYINSTALLER_LOG_LEVEL', 'WARN')}", f"--name={output_filename}", diff --git a/builder/pyinstaller_build_ubuntu.py b/builder/pyinstaller_build_ubuntu.py index 2b99122c..be6cc283 100644 --- a/builder/pyinstaller_build_ubuntu.py +++ b/builder/pyinstaller_build_ubuntu.py @@ -52,6 +52,8 @@ [ "--add-data=LICENSE:.", "--add-data=README.md:.", + f"--add-data={Path(__file__).parent / 'build' / 'tldextract_cache'/ '.suffix_cache'}:tldextract/.suffix_cache", + f"--add-data={Path(__file__).parent / 'build' / 'tldextract_cache'/'.tld_set_snapshot'}:tldextract/", f"--add-data={package_folder.joinpath('shortcuts/shortcut_freedesktop.template').resolve()}:shortcuts/", f"--log-level={getenv('PYINSTALLER_LOG_LEVEL', 'WARN')}", f"--name={output_filename}", diff --git a/builder/pyinstaller_build_windows.py b/builder/pyinstaller_build_windows.py index 71f59728..13866393 100644 --- a/builder/pyinstaller_build_windows.py +++ b/builder/pyinstaller_build_windows.py @@ -50,6 +50,8 @@ [ "--add-data=LICENSE:.", "--add-data=README.md:.", + f"--add-data={Path(__file__).parent / 'build' / 'tldextract_cache'/ '.suffix_cache'}:tldextract/.suffix_cache", + f"--add-data={Path(__file__).parent / 'build' / 'tldextract_cache'/'.tld_set_snapshot'}:tldextract/", # "--clean", f"--icon={package_folder.parent.resolve()}/docs/static/logo_qdt.ico", f"--log-level={getenv('PYINSTALLER_LOG_LEVEL', 'WARN')}", diff --git a/builder/tldextract_update.py b/builder/tldextract_update.py new file mode 100644 index 00000000..1a4786b1 --- /dev/null +++ b/builder/tldextract_update.py @@ -0,0 +1,83 @@ +#! python3 # noqa: E265 + +""" + Create tldextract update cache folder + +""" + +# ############################################################################# +# ########## Libraries ############# +# ################################## + +# Standard library +import argparse +import sys +import urllib.request +from os import W_OK, access, path +from pathlib import Path + +# 3rd party +from tldextract import TLDExtract + +sys.path.insert(0, path.abspath(r".")) + + +# ############################################################################# +# ########### MAIN ################# +# ################################## + + +def run(): + """Minimal CLI to generate a tldextract cache. + + :raises PermissionError: if output directory already exists but it's not writable + :raises SystemExit: in case of user abort + + :example: + + .. code-block:: bash + + python tldextract_update.py + """ + # variables + script_path = Path(__file__).parent + + # cli parser arguments + parser = argparse.ArgumentParser( + epilog=("tdlextract cache are created in output folder") + ) + parser.add_argument( + "-o", + "--output", + default=script_path / "build" / "tldextract_cache", + help="tld extract cache output folder", + type=Path, + ) + + args = parser.parse_args() + + try: + # check output directory + output_dir = Path(args.output) + if output_dir.exists() and not access(output_dir, W_OK): + raise PermissionError(output_dir.resolve()) + + output_dir.mkdir(exist_ok=True, parents=True) + + tld_extract = TLDExtract(str(output_dir / ".suffix_cache")) + tld_extract.update(True) + + urllib.request.urlretrieve( + "https://publicsuffix.org/list/public_suffix_list.dat", + output_dir / ".tld_set_snapshot", + ) + + # log user + print(f"tldextract cache written to: {output_dir.resolve()}") + except KeyboardInterrupt: + raise SystemExit("Aborted by user request.") + + +# Stand alone execution +if __name__ == "__main__": + run() diff --git a/docs/development/packaging.md b/docs/development/packaging.md index 6e09c614..bcc64321 100644 --- a/docs/development/packaging.md +++ b/docs/development/packaging.md @@ -14,6 +14,9 @@ The output binary and all embedded dependencies is located into a subfolder name # Generates MS Version Info python .\builder\version_info_templater.py +# Generates tldextract cache +python .\builder\tldextract_update.py + # Generates MS Executable python -O .\builder\pyinstaller_build_windows.py ``` @@ -29,6 +32,8 @@ To run it, double-click on the executable file (*.exe) loated into `dist` folder ```sh # Generates binary executable python -O ./builder/pyinstaller_build_ubuntu.py +# Generates tldextract cache +python ./builder/tldextract_update.py # make it runnable chmod +x dist/QGISDeploymentToolbelt_* ``` From 6b4698c64b48e2b28a78dc3a628e6e008f6dae17 Mon Sep 17 00:00:00 2001 From: jmkerloch Date: Mon, 7 Oct 2024 10:53:33 +0200 Subject: [PATCH 2/2] feat(builder): call tldextract in build scripts --- .github/workflows/build_release.yml | 8 -------- builder/pyinstaller_build_macos.py | 3 +++ builder/pyinstaller_build_ubuntu.py | 3 +++ builder/pyinstaller_build_windows.py | 3 +++ builder/tldextract_update.py | 6 +----- docs/development/packaging.md | 5 ----- 6 files changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml index f7c60811..c0cac18a 100644 --- a/.github/workflows/build_release.yml +++ b/.github/workflows/build_release.yml @@ -92,9 +92,6 @@ jobs: - name: Install project as a package run: python -m pip install -e . - - name: Generates tldextract cache - run: python -O ./builder/tldextract_update.py - - name: Generates Executable run: python -O ./builder/pyinstaller_build_macos.py @@ -129,8 +126,6 @@ jobs: - name: Install project as a package run: python -m pip install -e . - - name: Generates tldextract cache - run: python -O ./builder/tldextract_update.py - name: Generates Executable run: python -O ./builder/pyinstaller_build_ubuntu.py @@ -170,9 +165,6 @@ jobs: - name: Generates MS Version Info run: python .\builder\version_info_templater.py - - name: Generates tldextract cache - run: python -O .\builder\tldextract_update.py - - name: Generates MS Executable run: python -O .\builder\pyinstaller_build_windows.py diff --git a/builder/pyinstaller_build_macos.py b/builder/pyinstaller_build_macos.py index 8b5099a5..66c7e468 100644 --- a/builder/pyinstaller_build_macos.py +++ b/builder/pyinstaller_build_macos.py @@ -20,6 +20,7 @@ # package sys.path.insert(0, str(Path(".").resolve())) +from builder import tldextract_update from qgis_deployment_toolbelt import __about__ # noqa: E402 # ############################################################################# @@ -49,6 +50,8 @@ mac_os_version, _, _ = platform.mac_ver() mac_os_version = "-".join(mac_os_version.split(".")[:2]) +tldextract_update.run() + PyInstaller.__main__.run( [ "--add-data=LICENSE:.", diff --git a/builder/pyinstaller_build_ubuntu.py b/builder/pyinstaller_build_ubuntu.py index be6cc283..a4ea9263 100644 --- a/builder/pyinstaller_build_ubuntu.py +++ b/builder/pyinstaller_build_ubuntu.py @@ -21,6 +21,7 @@ # package sys.path.insert(0, str(Path(".").resolve())) +from builder import tldextract_update from qgis_deployment_toolbelt import __about__ # noqa: E402 # ############################################################################# @@ -48,6 +49,8 @@ ) package_folder = Path("qgis_deployment_toolbelt") +tldextract_update.run() + PyInstaller.__main__.run( [ "--add-data=LICENSE:.", diff --git a/builder/pyinstaller_build_windows.py b/builder/pyinstaller_build_windows.py index 13866393..33ae6074 100644 --- a/builder/pyinstaller_build_windows.py +++ b/builder/pyinstaller_build_windows.py @@ -20,6 +20,7 @@ # package sys.path.insert(0, str(Path(".").resolve())) +from builder import tldextract_update from qgis_deployment_toolbelt import __about__ # noqa: E402 # ############################################################################# @@ -46,6 +47,8 @@ ) package_folder = Path("qgis_deployment_toolbelt") +tldextract_update.run() + PyInstaller.__main__.run( [ "--add-data=LICENSE:.", diff --git a/builder/tldextract_update.py b/builder/tldextract_update.py index 1a4786b1..abee2a95 100644 --- a/builder/tldextract_update.py +++ b/builder/tldextract_update.py @@ -11,17 +11,13 @@ # Standard library import argparse -import sys import urllib.request -from os import W_OK, access, path +from os import W_OK, access from pathlib import Path # 3rd party from tldextract import TLDExtract -sys.path.insert(0, path.abspath(r".")) - - # ############################################################################# # ########### MAIN ################# # ################################## diff --git a/docs/development/packaging.md b/docs/development/packaging.md index bcc64321..6e09c614 100644 --- a/docs/development/packaging.md +++ b/docs/development/packaging.md @@ -14,9 +14,6 @@ The output binary and all embedded dependencies is located into a subfolder name # Generates MS Version Info python .\builder\version_info_templater.py -# Generates tldextract cache -python .\builder\tldextract_update.py - # Generates MS Executable python -O .\builder\pyinstaller_build_windows.py ``` @@ -32,8 +29,6 @@ To run it, double-click on the executable file (*.exe) loated into `dist` folder ```sh # Generates binary executable python -O ./builder/pyinstaller_build_ubuntu.py -# Generates tldextract cache -python ./builder/tldextract_update.py # make it runnable chmod +x dist/QGISDeploymentToolbelt_* ```