diff --git a/packages/dsw-command-queue/CHANGELOG.md b/packages/dsw-command-queue/CHANGELOG.md index 390b24bb..0c21529f 100644 --- a/packages/dsw-command-queue/CHANGELOG.md +++ b/packages/dsw-command-queue/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.27.0] + +Released for version consistency with other DSW tools. + ## [3.26.1] Released for version consistency with other DSW tools. @@ -132,3 +136,4 @@ Released for version consistency with other DSW tools. [3.25.0]: /../../tree/v3.25.0 [3.26.0]: /../../tree/v3.26.0 [3.26.1]: /../../tree/v3.26.1 +[3.27.0]: /../../tree/v3.27.0 diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml index 6467f3fc..ba08c3d3 100644 --- a/packages/dsw-command-queue/pyproject.toml +++ b/packages/dsw-command-queue/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-command-queue' -version = "3.26.1" +version = "3.27.0" description = 'Library for working with command queue and persistent commands' readme = 'README.md' keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing'] @@ -25,7 +25,7 @@ classifiers = [ requires-python = '>=3.10, <4' dependencies = [ # DSW - "dsw-database==3.26.1", + "dsw-database==3.27.0", ] [project.urls] diff --git a/packages/dsw-command-queue/requirements.txt b/packages/dsw-command-queue/requirements.txt index 31089512..d307ded8 100644 --- a/packages/dsw-command-queue/requirements.txt +++ b/packages/dsw-command-queue/requirements.txt @@ -1,5 +1,5 @@ -psycopg==3.1.9 -psycopg-binary==3.1.9 +psycopg==3.1.10 +psycopg-binary==3.1.10 PyYAML==6.0.1 tenacity==8.2.2 typing_extensions==4.7.1 diff --git a/packages/dsw-config/CHANGELOG.md b/packages/dsw-config/CHANGELOG.md index d6139a40..ad462cde 100644 --- a/packages/dsw-config/CHANGELOG.md +++ b/packages/dsw-config/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.27.0] + +Released for version consistency with other DSW tools. + ## [3.26.1] Released for version consistency with other DSW tools. @@ -140,3 +144,4 @@ Released for version consistency with other DSW tools. [3.25.0]: /../../tree/v3.25.0 [3.26.0]: /../../tree/v3.26.0 [3.26.1]: /../../tree/v3.26.1 +[3.27.0]: /../../tree/v3.27.0 diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml index f8283f3c..6ed23146 100644 --- a/packages/dsw-config/pyproject.toml +++ b/packages/dsw-config/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-config' -version = "3.26.1" +version = "3.27.0" description = 'Library for DSW config manipulation' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-config/requirements.txt b/packages/dsw-config/requirements.txt index 62c7d059..80b9f9ae 100644 --- a/packages/dsw-config/requirements.txt +++ b/packages/dsw-config/requirements.txt @@ -1,4 +1,4 @@ certifi==2023.7.22 PyYAML==6.0.1 -sentry-sdk==1.28.1 -urllib3==1.26.16 +sentry-sdk==1.29.2 +urllib3==2.0.4 diff --git a/packages/dsw-data-seeder/CHANGELOG.md b/packages/dsw-data-seeder/CHANGELOG.md index 193c9052..46a8e6c1 100644 --- a/packages/dsw-data-seeder/CHANGELOG.md +++ b/packages/dsw-data-seeder/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.27.0] + +Released for version consistency with other DSW tools. + ## [3.26.1] Released for version consistency with other DSW tools. @@ -180,3 +184,4 @@ Released for version consistency with other DSW tools. [3.25.0]: /../../tree/v3.25.0 [3.26.0]: /../../tree/v3.26.0 [3.26.1]: /../../tree/v3.26.1 +[3.27.0]: /../../tree/v3.27.0 diff --git a/packages/dsw-data-seeder/dsw/data_seeder/consts.py b/packages/dsw-data-seeder/dsw/data_seeder/consts.py index 78a1a184..d62fc27e 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/consts.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/consts.py @@ -6,4 +6,4 @@ DEFAULT_PLACEHOLDER = '<<|APP-ID|>>' NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'dsw-data-seeder' -VERSION = '3.26.1' +VERSION = '3.27.0' diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml index cb5cbab5..943d155a 100644 --- a/packages/dsw-data-seeder/pyproject.toml +++ b/packages/dsw-data-seeder/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-data-seeder' -version = "3.26.1" +version = "3.27.0" description = 'Worker for seeding DSW data' readme = 'README.md' keywords = ['data', 'database', 'seed', 'storage'] @@ -29,10 +29,10 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - "dsw-command-queue==3.26.1", - "dsw-config==3.26.1", - "dsw-database==3.26.1", - "dsw-storage==3.26.1", + "dsw-command-queue==3.27.0", + "dsw-config==3.27.0", + "dsw-database==3.27.0", + "dsw-storage==3.27.0", ] [project.urls] diff --git a/packages/dsw-data-seeder/requirements.txt b/packages/dsw-data-seeder/requirements.txt index 7a5f3e4d..8fc81f4d 100644 --- a/packages/dsw-data-seeder/requirements.txt +++ b/packages/dsw-data-seeder/requirements.txt @@ -1,13 +1,13 @@ certifi==2023.7.22 -click==8.1.6 +click==8.1.7 minio==7.1.15 python-dateutil==2.8.2 -psycopg==3.1.9 -psycopg-binary==3.1.9 +psycopg==3.1.10 +psycopg-binary==3.1.10 PyYAML==6.0.1 -sentry-sdk==1.28.1 +sentry-sdk==1.29.2 six==1.16.0 tenacity==8.2.2 typing_extensions==4.7.1 tzdata==2023.3 -urllib3==1.26.16 +urllib3==2.0.4 diff --git a/packages/dsw-database/CHANGELOG.md b/packages/dsw-database/CHANGELOG.md index 9dac025c..e7a05934 100644 --- a/packages/dsw-database/CHANGELOG.md +++ b/packages/dsw-database/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.27.0] + +Released for version consistency with other DSW tools. + ## [3.26.1] Released for version consistency with other DSW tools. @@ -147,3 +151,4 @@ Released for version consistency with other DSW tools. [3.25.1]: /../../tree/v3.25.1 [3.26.0]: /../../tree/v3.26.0 [3.26.1]: /../../tree/v3.26.1 +[3.27.0]: /../../tree/v3.27.0 diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml index e67fe3e2..3502a183 100644 --- a/packages/dsw-database/pyproject.toml +++ b/packages/dsw-database/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-database' -version = "3.26.1" +version = "3.27.0" description = 'Library for managing DSW database' readme = 'README.md' keywords = ['dsw', 'database'] @@ -26,7 +26,7 @@ dependencies = [ 'psycopg[binary]', 'tenacity', # DSW - "dsw-config==3.26.1", + "dsw-config==3.27.0", ] [project.urls] diff --git a/packages/dsw-database/requirements.txt b/packages/dsw-database/requirements.txt index 31089512..d307ded8 100644 --- a/packages/dsw-database/requirements.txt +++ b/packages/dsw-database/requirements.txt @@ -1,5 +1,5 @@ -psycopg==3.1.9 -psycopg-binary==3.1.9 +psycopg==3.1.10 +psycopg-binary==3.1.10 PyYAML==6.0.1 tenacity==8.2.2 typing_extensions==4.7.1 diff --git a/packages/dsw-document-worker/CHANGELOG.md b/packages/dsw-document-worker/CHANGELOG.md index 0946c736..2c764e5e 100644 --- a/packages/dsw-document-worker/CHANGELOG.md +++ b/packages/dsw-document-worker/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.27.0] + +Released for version consistency with other DSW tools. + ## [3.26.1] Released for version consistency with other DSW tools. @@ -171,3 +175,4 @@ Released for version consistency with other DSW tools. [3.25.0]: /../../tree/v3.25.0 [3.26.0]: /../../tree/v3.26.0 [3.26.1]: /../../tree/v3.26.1 +[3.27.0]: /../../tree/v3.27.0 diff --git a/packages/dsw-document-worker/dsw/document_worker/consts.py b/packages/dsw-document-worker/dsw/document_worker/consts.py index 0964a007..024b0970 100644 --- a/packages/dsw-document-worker/dsw/document_worker/consts.py +++ b/packages/dsw-document-worker/dsw/document_worker/consts.py @@ -6,7 +6,7 @@ EXIT_SUCCESS = 0 NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'docworker' -VERSION = '3.26.1' +VERSION = '3.27.0' class DocumentState: diff --git a/packages/dsw-document-worker/pyproject.toml b/packages/dsw-document-worker/pyproject.toml index 4bf1a046..d42379b8 100644 --- a/packages/dsw-document-worker/pyproject.toml +++ b/packages/dsw-document-worker/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-document-worker' -version = "3.26.1" +version = "3.27.0" description = 'Worker for assembling and transforming documents' readme = 'README.md' keywords = ['documents', 'generation', 'jinja2', 'pandoc', 'worker'] @@ -40,10 +40,10 @@ dependencies = [ 'weasyprint', 'XlsxWriter', # DSW - "dsw-command-queue==3.26.1", - "dsw-config==3.26.1", - "dsw-database==3.26.1", - "dsw-storage==3.26.1", + "dsw-command-queue==3.27.0", + "dsw-config==3.27.0", + "dsw-database==3.27.0", + "dsw-storage==3.27.0", ] [project.urls] diff --git a/packages/dsw-document-worker/requirements.txt b/packages/dsw-document-worker/requirements.txt index bb985f1b..c4a90712 100644 --- a/packages/dsw-document-worker/requirements.txt +++ b/packages/dsw-document-worker/requirements.txt @@ -2,9 +2,9 @@ Brotli==1.0.9 certifi==2023.7.22 cffi==1.15.1 charset-normalizer==3.2.0 -click==8.1.6 +click==8.1.7 cssselect2==0.7.0 -fonttools==4.41.1 +fonttools==4.42.0 html5lib==1.1 idna==3.4 isodate==0.6.1 @@ -15,9 +15,9 @@ mdx-breakless-lists==1.0.1 minio==7.1.15 pathvalidate==3.1.0 pdfrw==0.4 -Pillow==9.5.0 -psycopg==3.1.9 -psycopg-binary==3.1.9 +Pillow==10.0.0 +psycopg==3.1.10 +psycopg-binary==3.1.10 pycparser==2.21 pydyf==0.7.0 pyparsing==3.1.1 @@ -25,17 +25,17 @@ pyphen==0.14.0 python-dateutil==2.8.2 python-slugify==8.0.1 PyYAML==6.0.1 -rdflib==6.3.2 +rdflib==7.0.0 rdflib-jsonld==0.6.2 requests==2.31.0 -sentry-sdk==1.28.1 +sentry-sdk==1.29.2 six==1.16.0 tenacity==8.2.2 text-unidecode==1.3 tinycss2==1.2.1 typing_extensions==4.7.1 tzdata==2023.3 -urllib3==1.26.16 +urllib3==2.0.4 weasyprint==59.0 webencodings==0.5.1 XlsxWriter==3.1.2 diff --git a/packages/dsw-mailer/CHANGELOG.md b/packages/dsw-mailer/CHANGELOG.md index 5ed38578..6fa894c6 100644 --- a/packages/dsw-mailer/CHANGELOG.md +++ b/packages/dsw-mailer/CHANGELOG.md @@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.27.0] + +### Added + +- Notification emails about API keys + ## [3.26.1] ### Fixed @@ -155,3 +161,4 @@ Released for version consistency with other DSW tools. [3.25.0]: /../../tree/v3.25.0 [3.26.0]: /../../tree/v3.26.0 [3.26.1]: /../../tree/v3.26.1 +[3.27.0]: /../../tree/v3.27.0 diff --git a/packages/dsw-mailer/dsw/mailer/consts.py b/packages/dsw-mailer/dsw/mailer/consts.py index a615ac94..0a3883b8 100644 --- a/packages/dsw-mailer/dsw/mailer/consts.py +++ b/packages/dsw-mailer/dsw/mailer/consts.py @@ -5,4 +5,4 @@ DEFAULT_ENCODING = 'utf-8' NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'dsw-mailer' -VERSION = '3.26.1' +VERSION = '3.27.0' diff --git a/packages/dsw-mailer/dsw/mailer/templates.py b/packages/dsw-mailer/dsw/mailer/templates.py index 1cd100fa..507118cc 100644 --- a/packages/dsw-mailer/dsw/mailer/templates.py +++ b/packages/dsw-mailer/dsw/mailer/templates.py @@ -1,9 +1,11 @@ +import datetime +import dateutil.parser import jinja2 import json import logging import pathlib -from typing import Optional +from typing import Optional, Union from .config import MailerConfig, MailConfig from .consts import DEFAULT_ENCODING @@ -68,8 +70,14 @@ def __init__(self, cfg: MailerConfig, workdir: pathlib.Path): extensions=['jinja2.ext.do'], ) self.templates = dict() # type: dict[str, MailTemplate] + self._set_filters() self._load_templates() + def _set_filters(self): + self.j2_env.filters.update({ + 'datetime_format': datetime_format, + }) + def _load_jinja2(self, file_path: pathlib.Path) -> Optional[jinja2.Template]: if file_path.exists() and file_path.is_file(): return self.j2_env.get_template( @@ -153,3 +161,11 @@ def render(self, rq: MessageRequest, cfg: MailConfig) -> MailMessage: mail_name=used_cfg.name, mail_from=used_cfg.email, ) + + +def datetime_format(iso_timestamp: Union[None, datetime.datetime, str], fmt: str): + if iso_timestamp is None: + return '' + if not isinstance(iso_timestamp, datetime.datetime): + iso_timestamp = dateutil.parser.isoparse(iso_timestamp) + return iso_timestamp.strftime(fmt) diff --git a/packages/dsw-mailer/pyproject.toml b/packages/dsw-mailer/pyproject.toml index 4f313d26..45a79be3 100644 --- a/packages/dsw-mailer/pyproject.toml +++ b/packages/dsw-mailer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-mailer' -version = "3.26.1" +version = "3.27.0" description = 'Worker for sending email notifications' readme = 'README.md' keywords = ['email', 'jinja2', 'notification', 'template'] @@ -31,9 +31,9 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - "dsw-command-queue==3.26.1", - "dsw-config==3.26.1", - "dsw-database==3.26.1", + "dsw-command-queue==3.27.0", + "dsw-config==3.27.0", + "dsw-database==3.27.0", ] [project.urls] diff --git a/packages/dsw-mailer/requirements.txt b/packages/dsw-mailer/requirements.txt index e6c71289..eb82ca2c 100644 --- a/packages/dsw-mailer/requirements.txt +++ b/packages/dsw-mailer/requirements.txt @@ -1,17 +1,17 @@ certifi==2023.7.22 -click==8.1.6 +click==8.1.7 dkimpy==1.1.5 -dnspython==2.4.1 +dnspython==2.4.2 Jinja2==3.1.2 MarkupSafe==2.1.3 pathvalidate==3.1.0 -psycopg==3.1.9 -psycopg-binary==3.1.9 +psycopg==3.1.10 +psycopg-binary==3.1.10 python-dateutil==2.8.2 PyYAML==6.0.1 -sentry-sdk==1.28.1 +sentry-sdk==1.29.2 six==1.16.0 tenacity==8.2.2 typing_extensions==4.7.1 tzdata==2023.3 -urllib3==1.26.16 +urllib3==2.0.4 diff --git a/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.html.j2 b/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.html.j2 new file mode 100644 index 00000000..97c18a5f --- /dev/null +++ b/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.html.j2 @@ -0,0 +1,92 @@ +{% extends '_common/layout.html.j2' %} +{% block body %} + + + + + + +
+ + + + + + + + +
+

+ Dear {{ ctx.userFirstName }},

+ A new API key named {{ ctx.tokenName }} has been created for your account.
+ It will expire on {{ ctx.tokenExpiresAt|datetime_format("%B %-d, %Y (at %H:%M %Z)") }}. +

+
+ + +
+ + + + + + +
+ + + + + + +
+ Manage API Keys +
+
+ + + + + + +
+ + + + + + + + +
+

+ {% if ctx.supportEmail %} + If you did not create this API key or encounter any issues, please contact us via {{ ctx.supportEmail }}. + {% endif %}

+ Thank you for using the DSW!

+ {% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %} +

+
+ + +
+{% endblock %} diff --git a/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.json b/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.json new file mode 100644 index 00000000..cd4f58af --- /dev/null +++ b/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.json @@ -0,0 +1,16 @@ +{ + "id": "wizard:apiKeyCreated", + "subject": "A new API key has been created", + "subjectPrefix": true, + "defaultSenderName": "DSW", + "parts": [ + { + "type": "html", + "file": "message.html.j2" + }, + { + "type": "plain", + "file": "message.txt.j2" + } + ] +} diff --git a/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.txt.j2 b/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.txt.j2 new file mode 100644 index 00000000..7f802588 --- /dev/null +++ b/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.txt.j2 @@ -0,0 +1,12 @@ +Dear {{ ctx.userFirstName }}, + +A new API key named "{{ ctx.tokenName }}" has been created for your account. +It will expire on {{ ctx.tokenExpiresAt|datetime_format("%B %-d, %Y (at %H:%M %Z)") }}. + +You can manage your API keys here: {{ ctx.clientUrl }}/users/edit/current/api-keys + +{% if ctx.supportEmail %} +If you did not initiate this login attempt, please contact us via {{ ctx.supportEmail }}. +{% endif %} + +{% include '_common/footer.txt.j2' %} diff --git a/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.html.j2 b/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.html.j2 new file mode 100644 index 00000000..f68f235d --- /dev/null +++ b/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.html.j2 @@ -0,0 +1,91 @@ +{% extends '_common/layout.html.j2' %} +{% block body %} + + + + + + +
+ + + + + + + + +
+

+ Dear {{ ctx.userFirstName }},

+ Your API key, identified as {{ ctx.tokenName }}, is scheduled to expire on {{ ctx.tokenExpiresAt|datetime_format("%B %-d, %Y (at %H:%M %Z)") }}. +

+
+ + +
+ + + + + + +
+ + + + + + +
+ Manage API Keys +
+
+ + + + + + +
+ + + + + + + + +
+

+ {% if ctx.supportEmail %} + If you encounter any issues, please contact us via {{ ctx.supportEmail }}. + {% endif %}

+ Thank you for using the DSW!

+ {% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %} +

+
+ + +
+{% endblock %} diff --git a/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.json b/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.json new file mode 100644 index 00000000..840726f4 --- /dev/null +++ b/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.json @@ -0,0 +1,16 @@ +{ + "id": "wizard:apiKeyExpiration", + "subject": "Your API key will expire soon", + "subjectPrefix": true, + "defaultSenderName": "DSW", + "parts": [ + { + "type": "html", + "file": "message.html.j2" + }, + { + "type": "plain", + "file": "message.txt.j2" + } + ] +} diff --git a/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.txt.j2 b/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.txt.j2 new file mode 100644 index 00000000..595c72cf --- /dev/null +++ b/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.txt.j2 @@ -0,0 +1,11 @@ +Dear {{ ctx.userFirstName }}, + +Your API key, identified as {{ ctx.tokenName }}, is scheduled to expire on {{ ctx.tokenExpiresAt|datetime_format("%B %-d, %Y (at %H:%M %Z)") }}. + +You can manage your API keys here: {{ ctx.clientUrl }}/users/edit/current/api-keys + +{% if ctx.supportEmail %} +If you did not initiate this login attempt, please contact us via {{ ctx.supportEmail }}. +{% endif %} + +{% include '_common/footer.txt.j2' %} diff --git a/packages/dsw-models/CHANGELOG.md b/packages/dsw-models/CHANGELOG.md index f1b8f62b..c8c9ff97 100644 --- a/packages/dsw-models/CHANGELOG.md +++ b/packages/dsw-models/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.27.0] + +Released for version consistency with other DSW tools. + ## [3.26.1] Released for version consistency with other DSW tools. @@ -65,3 +69,4 @@ Released for version consistency with other DSW tools. [3.25.0]: /../../tree/v3.25.0 [3.26.0]: /../../tree/v3.26.0 [3.26.1]: /../../tree/v3.26.1 +[3.27.0]: /../../tree/v3.27.0 diff --git a/packages/dsw-models/pyproject.toml b/packages/dsw-models/pyproject.toml index 973993d3..794a1027 100644 --- a/packages/dsw-models/pyproject.toml +++ b/packages/dsw-models/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-models' -version = "3.26.1" +version = "3.27.0" description = 'Library with DSW models and basic IO operations' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-models/requirements.txt b/packages/dsw-models/requirements.txt index 0d386161..e69de29b 100644 --- a/packages/dsw-models/requirements.txt +++ b/packages/dsw-models/requirements.txt @@ -1 +0,0 @@ -# No requirements at this point diff --git a/packages/dsw-storage/CHANGELOG.md b/packages/dsw-storage/CHANGELOG.md index 7ca3eb05..e7984041 100644 --- a/packages/dsw-storage/CHANGELOG.md +++ b/packages/dsw-storage/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.27.0] + +Released for version consistency with other DSW tools. + ## [3.26.1] Released for version consistency with other DSW tools. @@ -127,3 +131,4 @@ Released for version consistency with other DSW tools. [3.25.0]: /../../tree/v3.25.0 [3.26.0]: /../../tree/v3.26.0 [3.26.1]: /../../tree/v3.26.1 +[3.27.0]: /../../tree/v3.27.0 diff --git a/packages/dsw-storage/pyproject.toml b/packages/dsw-storage/pyproject.toml index 3c2fd149..03cdc665 100644 --- a/packages/dsw-storage/pyproject.toml +++ b/packages/dsw-storage/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-storage' -version = "3.26.1" +version = "3.27.0" description = 'Library for managing DSW S3 storage' readme = 'README.md' keywords = ['dsw', 's3', 'bucket', 'storage'] @@ -26,7 +26,7 @@ dependencies = [ 'minio', 'tenacity', # DSW - "dsw-config==3.26.1", + "dsw-config==3.27.0", ] [project.urls] diff --git a/packages/dsw-storage/requirements.txt b/packages/dsw-storage/requirements.txt index 1dc1a2bc..58f3436c 100644 --- a/packages/dsw-storage/requirements.txt +++ b/packages/dsw-storage/requirements.txt @@ -2,4 +2,4 @@ certifi==2023.7.22 minio==7.1.15 PyYAML==6.0.1 tenacity==8.2.2 -urllib3==1.26.16 +urllib3==2.0.4 diff --git a/packages/dsw-tdk/CHANGELOG.md b/packages/dsw-tdk/CHANGELOG.md index 48450cc1..7d0d660c 100644 --- a/packages/dsw-tdk/CHANGELOG.md +++ b/packages/dsw-tdk/CHANGELOG.md @@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.27.0] + +### Changed + +- Removed legacy support for authentication using credentials + ## [3.26.1] Released for version consistency with other DSW tools. @@ -340,3 +346,4 @@ Initial DSW Template Development Kit (versioned as part of the [DSW platform](ht [3.25.0]: /../../tree/v3.25.0 [3.26.0]: /../../tree/v3.26.0 [3.26.1]: /../../tree/v3.26.1 +[3.27.0]: /../../tree/v3.27.0 diff --git a/packages/dsw-tdk/dsw/tdk/api_client.py b/packages/dsw-tdk/dsw/tdk/api_client.py index d90d709d..11e8877f 100644 --- a/packages/dsw-tdk/dsw/tdk/api_client.py +++ b/packages/dsw-tdk/dsw/tdk/api_client.py @@ -81,7 +81,7 @@ def _check_status(r: aiohttp.ClientResponse, expected_status): f'{r.reason} (expecting {expected_status})' ) - def __init__(self, api_url: str, session=None): + def __init__(self, api_url: str, api_key: str, session=None): """ Exception representing communication error with DSW. @@ -90,7 +90,7 @@ def __init__(self, api_url: str, session=None): session (aiohttp.ClientSession): Optional custom session for HTTP communication. """ self.api_url = api_url - self.token = None + self.token = api_key self.session = session or aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) @property diff --git a/packages/dsw-tdk/dsw/tdk/cli.py b/packages/dsw-tdk/dsw/tdk/cli.py index 7e26a00a..3a02fa4c 100644 --- a/packages/dsw-tdk/dsw/tdk/cli.py +++ b/packages/dsw-tdk/dsw/tdk/cli.py @@ -1,5 +1,4 @@ import asyncio -import sys import click # type: ignore import datetime @@ -96,6 +95,10 @@ def rectify_url(ctx, param, value) -> str: return value.rstrip('/') +def rectify_key(ctx, param, value) -> str: + return value.strip() + + class ClickLogger(logging.Logger): NAME = 'DSW-TDK-CLI' @@ -169,6 +172,7 @@ class CLIContext: def __init__(self): self.logger = ClickLogger.default() + self.dot_env_file = None def debug_mode(self): self.logger.show_timestamp = True @@ -178,37 +182,6 @@ def quiet_mode(self): self.logger.muted = True -class APICredentials: - - def __init__(self, username, password, api_key): - self.username = username - self.password = password - self.api_key = api_key - - def check(self): - if self.api_key is not None: - return - if self.username is not None and self.password is not None: - ClickPrinter.warning('Using username/password credentials, ' - 'consider switching to API keys.') - ClickPrinter.warning('Username/password authentication will be ' - 'removed in 3.25 as deprecated.') - return False - ClickPrinter.failure('Invalid credentials entered! You need to provide ' - 'either API key or username/password credentials.') - sys.exit(1) - - def init_args(self): - if self.api_key is not None: - return { - 'api_key': self.api_key, - } - return { - 'username': self.username, - 'password': self.password, - } - - def interact_formats() -> Dict[str, FormatSpec]: add_format = click.confirm('Do you want to add a format?', default=True) formats = dict() # type: Dict[str, FormatSpec] @@ -273,13 +246,14 @@ def main(ctx, quiet, debug, dot_env): if pathlib.Path(dot_env).exists(): dotenv.load_dotenv(dotenv_path=dot_env) ctx.ensure_object(CLIContext) + ctx.obj.dot_env_file = dot_env if quiet: ctx.obj.quiet_mode() if debug: ctx.obj.debug_mode() -@main.command(help='Create a new DSW template project.', name='new') +@main.command(help='Create a new template project.', name='new') @click.argument('TEMPLATE-DIR', type=NEW_DIR_TYPE, default=None, required=False) @click.option('-f', '--force', is_flag=True, help='Overwrite any matching files.') @click.pass_context @@ -303,30 +277,25 @@ def new_template(ctx, template_dir, force): exit(1) -@main.command(help='Download template from DSW.', name='get') +@main.command(help='Download template from Wizard.', name='get') @click.argument('TEMPLATE-ID') @click.argument('TEMPLATE-DIR', type=NEW_DIR_TYPE, default=None, required=False) -@click.option('-s', '--api-server', metavar='API-URL', envvar='DSW_API', - prompt='URL of DSW API', help='URL of DSW server API.', callback=rectify_url) -@click.option('-u', '--username', envvar='DSW_USERNAME', metavar='EMAIL', default=None, - help='Admin username (email) for DSW instance.') -@click.option('-p', '--password', envvar='DSW_PASSWORD', metavar='PASSWORD', default=None, - help='Admin password for DSW instance.') -@click.option('-k', '--api-key', envvar='DSW_API_KEY', metavar='API-KEY', default=None, - help='API key for DSW instance.') +@click.option('-u', '--api-url', metavar='API-URL', envvar='DSW_API_URL', + prompt='API URL', help='URL of Wizard server API.', callback=rectify_url) +@click.option('-k', '--api-key', metavar='API-KEY', envvar='DSW_API_KEY', + prompt='API Key', help='API key for Wizard instance.', callback=rectify_key, + hide_input=True) @click.option('-f', '--force', is_flag=True, help='Overwrite any existing files.') @click.pass_context -def get_template(ctx, api_server, template_id, template_dir, username, password, api_key, force): +def get_template(ctx, api_url, template_id, template_dir, api_key, force): template_dir = pathlib.Path(template_dir or dir_from_id(template_id)) - creds = APICredentials(username=username, password=password, api_key=api_key) - creds.check() async def main_routine(): tdk = TDKCore(logger=ctx.obj.logger) template_type = 'unknown' zip_data = None try: - await tdk.init_client(api_url=api_server, **creds.init_args()) + await tdk.init_client(api_url=api_url, api_key=api_key) try: await tdk.load_remote(template_id=template_id) template_type = 'draft' @@ -367,23 +336,18 @@ async def main_routine(): loop.run_until_complete(main_routine()) -@main.command(help='Upload template to DSW.', name='put') +@main.command(help='Upload template to Wizard.', name='put') @click.argument('TEMPLATE-DIR', type=DIR_TYPE, default=CURRENT_DIR, required=False) -@click.option('-s', '--api-server', metavar='API-URL', envvar='DSW_API', - prompt='URL of DSW API', help='URL of DSW server API.', callback=rectify_url) -@click.option('-u', '--username', envvar='DSW_USERNAME', metavar='EMAIL', default=None, - help='Admin username (email) for DSW instance.') -@click.option('-p', '--password', envvar='DSW_PASSWORD', metavar='PASSWORD', default=None, - help='Admin password for DSW instance.') -@click.option('-k', '--api-key', envvar='DSW_API_KEY', metavar='API-KEY', default=None, - help='API key for DSW instance.') +@click.option('-u', '--api-url', metavar='API-URL', envvar='DSW_API_URL', + prompt='API URL', help='URL of Wizard server API.', callback=rectify_url) +@click.option('-k', '--api-key', metavar='API-KEY', envvar='DSW_API_KEY', + prompt='API Key', help='API key for Wizard instance.', callback=rectify_key, + hide_input=True) @click.option('-f', '--force', is_flag=True, help='Delete template if already exists.') @click.option('-w', '--watch', is_flag=True, help='Enter watch mode to continually upload changes.') @click.pass_context -def put_template(ctx, api_server, template_dir, username, password, api_key, force, watch): +def put_template(ctx, api_url, template_dir, api_key, force, watch): tdk = TDKCore(logger=ctx.obj.logger) - creds = APICredentials(username=username, password=password, api_key=api_key) - creds.check() async def watch_callback(changes): changes = list(changes) @@ -399,10 +363,10 @@ async def watch_callback(changes): async def main_routine(): load_local(tdk, template_dir) try: - await tdk.init_client(api_url=api_server, **creds.init_args()) + await tdk.init_client(api_url=api_url, api_key=api_key) await tdk.store_remote(force=force) ClickPrinter.success(f'Template {tdk.safe_project.safe_template.id} ' - f'uploaded to {api_server}') + f'uploaded to {api_url}') if watch: ClickPrinter.watch('Entering watch mode... (press Ctrl+C to abort)') @@ -427,7 +391,7 @@ async def main_routine(): loop.run_until_complete(main_routine()) -@main.command(help='Create ZIP package for DSW template.', name='package') +@main.command(help='Create ZIP package for a template.', name='package') @click.argument('TEMPLATE-DIR', type=DIR_TYPE, default=CURRENT_DIR, required=False) @click.option('-o', '--output', default='template.zip', type=click.Path(writable=True), show_default=True, help='Target package file.') @@ -446,7 +410,7 @@ def create_package(ctx, template_dir, output, force: bool): ClickPrinter.success(f'Package {filename} created') -@main.command(help='Extract DSW template from ZIP package', name='unpackage') +@main.command(help='Extract a template from ZIP package', name='unpackage') @click.argument('TEMPLATE-PACKAGE', type=FILE_READ_TYPE, required=False) @click.option('-o', '--output', type=NEW_DIR_TYPE, default=None, required=False, help='Target package file.') @@ -468,29 +432,23 @@ def extract_package(ctx, template_package, output, force: bool): ClickPrinter.success(f'Package {template_package} extracted') -@main.command(help='List templates from DSW via API.', name='list') -@click.option('-s', '--api-server', metavar='API-URL', envvar='DSW_API', - prompt='URL of DSW API', help='URL of DSW server API.', callback=rectify_url) -@click.option('-u', '--username', envvar='DSW_USERNAME', metavar='EMAIL', default=None, - help='Admin username (email) for DSW instance.') -@click.option('-p', '--password', envvar='DSW_PASSWORD', metavar='PASSWORD', default=None, - help='Admin password for DSW instance.') -@click.option('-k', '--api-key', envvar='DSW_API_KEY', metavar='API-KEY', default=None, - help='API key for DSW instance.') +@main.command(help='List templates from Wizard via API.', name='list') +@click.option('-u', '--api-url', metavar='API-URL', envvar='DSW_API_URL', + prompt='API URL', help='URL of Wizard server API.', callback=rectify_url) +@click.option('-k', '--api-key', metavar='API-KEY', envvar='DSW_API_KEY', + prompt='API Key', help='API key for Wizard instance.', callback=rectify_key, + hide_input=True) @click.option('--output-format', default=DEFAULT_LIST_FORMAT, metavar='FORMAT', help='Entry format string for printing.') @click.option('-r', '--released-only', is_flag=True, help='List only released templates') @click.option('-d', '--drafts-only', is_flag=True, help='List only template drafts') @click.pass_context -def list_templates(ctx, api_server, username, password, api_key, output_format: str, +def list_templates(ctx, api_url, api_key, output_format: str, released_only: bool, drafts_only: bool): - creds = APICredentials(username=username, password=password, api_key=api_key) - creds.check() - async def main_routine(): tdk = TDKCore(logger=ctx.obj.logger) try: - await tdk.init_client(api_url=api_server, **creds.init_args()) + await tdk.init_client(api_url=api_url, api_key=api_key) if released_only: templates = await tdk.list_remote_templates() for template in templates: @@ -520,7 +478,7 @@ async def main_routine(): loop.run_until_complete(main_routine()) -@main.command(help='Verify DSW template project.', name='verify') +@main.command(help='Verify a template project.', name='verify') @click.argument('TEMPLATE-DIR', type=DIR_TYPE, default=CURRENT_DIR, required=False) @click.pass_context def verify_template(ctx, template_dir): @@ -535,3 +493,28 @@ def verify_template(ctx, template_dir): click.echo('Found violations:') for err in errors: click.echo(f' - {err.field_name}: {err.message}') + + +@main.command(help='Create a .env file.', name='dot-env') +@click.argument('TEMPLATE-DIR', type=DIR_TYPE, default=CURRENT_DIR, required=False) +@click.option('-u', '--api-url', metavar='API-URL', envvar='DSW_API_URL', + prompt='API URL', help='URL of Wizard server API.', callback=rectify_url) +@click.option('-k', '--api-key', metavar='API-KEY', envvar='DSW_API_KEY', + prompt='API Key', help='API key for Wizard instance.', callback=rectify_key, + hide_input=True) +@click.option('-f', '--force', is_flag=True, help='Overwrite file if already exists.') +@click.pass_context +def create_dot_env(ctx, template_dir, api_url, api_key, force): + filename = ctx.obj.dot_env_file or '.env' + tdk = TDKCore(logger=ctx.obj.logger) + try: + tdk.create_dot_env( + output=pathlib.Path(template_dir) / filename, + force=force, + api_url=api_url, + api_key=api_key, + ) + except Exception as e: + ClickPrinter.failure('Failed to create dot-env file') + ClickPrinter.error(f'> {e}') + exit(1) diff --git a/packages/dsw-tdk/dsw/tdk/consts.py b/packages/dsw-tdk/dsw/tdk/consts.py index e941b298..9fcaaa48 100644 --- a/packages/dsw-tdk/dsw/tdk/consts.py +++ b/packages/dsw-tdk/dsw/tdk/consts.py @@ -3,7 +3,7 @@ import re APP = 'dsw-tdk' -VERSION = '3.26.1' +VERSION = '3.27.0' METAMODEL_VERSION = 11 REGEX_SEMVER = re.compile(r'^[0-9]+\.[0-9]+\.[0-9]+$') diff --git a/packages/dsw-tdk/dsw/tdk/core.py b/packages/dsw-tdk/dsw/tdk/core.py index d80082ba..18ee0896 100644 --- a/packages/dsw-tdk/dsw/tdk/core.py +++ b/packages/dsw-tdk/dsw/tdk/core.py @@ -15,7 +15,7 @@ from .api_client import DSWAPIClient, DSWCommunicationError from .consts import DEFAULT_ENCODING, REGEX_SEMVER from .model import TemplateProject, Template, TemplateFile, TemplateFileType -from .utils import UUIDGen +from .utils import UUIDGen, create_dot_env from .validation import ValidationError, TemplateValidator @@ -99,14 +99,9 @@ def safe_client(self) -> DSWAPIClient: raise RuntimeError('No DSW API client specified') return self.client - async def init_client(self, api_url: str, username: Optional[str] = None, - password: Optional[str] = None, api_key: Optional[str] = None): + async def init_client(self, api_url: str, api_key: str): self.logger.info(f'Connecting to {api_url}') - self.client = DSWAPIClient(api_url=api_url) - if api_key is not None: - self.client.token = api_key # type: ignore - if username is not None and password is not None: - await self.client.login(email=username, password=password) + self.client = DSWAPIClient(api_url=api_url, api_key=api_key) self.remote_version = await self.client.get_api_version() user = await self.client.get_current_user() self.logger.info(f'Successfully authenticated as {user["firstName"]} ' @@ -362,6 +357,17 @@ def extract_package(self, zip_data: bytes, template_dir: Optional[pathlib.Path], target_file.write_text(data=content, encoding=DEFAULT_ENCODING) self.logger.debug('Extracting package done') + def create_dot_env(self, output: pathlib.Path, force: bool, api_url: str, api_key: str): + if output.exists(): + if force: + self.logger.warning(f'Overwriting {output.as_posix()} (forced)') + else: + raise RuntimeError(f'File {output} already exists (not forced)') + output.write_text( + data=create_dot_env(api_url=api_url, api_key=api_key), + encoding=DEFAULT_ENCODING, + ) + async def watch_project(self, callback): async for changes in watchgod.awatch(self.safe_project.template_dir): await callback(( diff --git a/packages/dsw-tdk/dsw/tdk/templates/LICENSE.j2 b/packages/dsw-tdk/dsw/tdk/templates/LICENSE.j2 new file mode 100644 index 00000000..27cdf109 --- /dev/null +++ b/packages/dsw-tdk/dsw/tdk/templates/LICENSE.j2 @@ -0,0 +1 @@ +// TODO: fill text of your license ({{ template.license }}) diff --git a/packages/dsw-tdk/dsw/tdk/templates/env.j2 b/packages/dsw-tdk/dsw/tdk/templates/env.j2 new file mode 100644 index 00000000..f2f091bb --- /dev/null +++ b/packages/dsw-tdk/dsw/tdk/templates/env.j2 @@ -0,0 +1,2 @@ +DSW_API_URL={{ api_url|default("http://localhost:3000") }} +DSW_API_KEY={{ api_key|default("") }} diff --git a/packages/dsw-tdk/dsw/tdk/utils.py b/packages/dsw-tdk/dsw/tdk/utils.py index d4b11344..5ba26caa 100644 --- a/packages/dsw-tdk/dsw/tdk/utils.py +++ b/packages/dsw-tdk/dsw/tdk/utils.py @@ -2,7 +2,7 @@ import pathlib import uuid -from typing import List, Set +from typing import List, Set, Optional from .consts import DEFAULT_ENCODING, DEFAULT_README from .model import Template, TemplateFile, Format, Step, PackageFilter @@ -163,13 +163,33 @@ def build(self) -> Template: self.template.readme = readme self.template.tdk_config.readme_file = DEFAULT_README TemplateValidator.validate(self.template) + for format_spec in self._formats: content = j2_env.get_template('starter.j2').render(template=self.template) self.template.files[format_spec.filename] = TemplateFile( filename=format_spec.filename, content_type='text/plain', - content=content.encode(encoding=DEFAULT_ENCODING) + content=content.encode(encoding=DEFAULT_ENCODING), ) self.template.tdk_config.files.append(str(format_spec.filename)) self.template.allowed_packages.append(PackageFilter()) + + license_file = j2_env.get_template('LICENSE.j2').render(template=self.template) + self.template.tdk_config.files.append('LICENSE') + self.template.files['LICENSE'] = TemplateFile( + filename='LICENSE', + content_type='text/plain', + content=license_file.encode(encoding=DEFAULT_ENCODING), + ) + + self.template.files['.env'] = TemplateFile( + filename='.env', + content_type='text/plain', + content=create_dot_env().encode(encoding=DEFAULT_ENCODING), + ) + return self.template + + +def create_dot_env(api_url: Optional[str] = None, api_key: Optional[str] = None) -> str: + return j2_env.get_template('env.j2').render(api_url=api_url, api_key=api_key) diff --git a/packages/dsw-tdk/pyproject.toml b/packages/dsw-tdk/pyproject.toml index c16a8d68..467a75d1 100644 --- a/packages/dsw-tdk/pyproject.toml +++ b/packages/dsw-tdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-tdk' -version = "3.26.1" +version = "3.27.0" description = 'Data Stewardship Wizard Template Development Toolkit' readme = 'README.md' keywords = ['documents', 'dsw', 'jinja2', 'template', 'toolkit'] diff --git a/packages/dsw-tdk/requirements.txt b/packages/dsw-tdk/requirements.txt index 6f6d454f..14242f95 100644 --- a/packages/dsw-tdk/requirements.txt +++ b/packages/dsw-tdk/requirements.txt @@ -4,7 +4,7 @@ anyio==3.7.1 async-timeout==4.0.2 attrs==23.1.0 charset-normalizer==3.2.0 -click==8.1.6 +click==8.1.7 colorama==0.4.6 frozenlist==1.4.0 humanize==4.7.0 diff --git a/packages/dsw-tdk/tests/conftest.py b/packages/dsw-tdk/tests/conftest.py index da61dd88..0be5c69d 100644 --- a/packages/dsw-tdk/tests/conftest.py +++ b/packages/dsw-tdk/tests/conftest.py @@ -20,7 +20,7 @@ @pytest.fixture(scope='session') def dsw_api_url(): - value = os.environ.get('DSW_API', 'http://localhost:3000') + value = os.environ.get('DSW_API_URL', 'http://localhost:3000') return value @@ -33,7 +33,7 @@ def dsw_api_key(): @pytest.fixture(scope='session') def dsw_env(dsw_api_url: str, dsw_api_key: str): return { - 'DSW_API': dsw_api_url, + 'DSW_API_URL': dsw_api_url, 'DSW_API_KEY': dsw_api_key, } diff --git a/packages/dsw-tdk/tests/test_cmd_dot-env.py b/packages/dsw-tdk/tests/test_cmd_dot-env.py new file mode 100644 index 00000000..47ae18e0 --- /dev/null +++ b/packages/dsw-tdk/tests/test_cmd_dot-env.py @@ -0,0 +1,70 @@ +import pathlib + +import click.testing + +from dsw.tdk import main + + +def test_creates_dot_env(tmp_path: pathlib.Path): + runner = click.testing.CliRunner() + inputs = ['https://example.com', 'mySecretKey'] + with runner.isolated_filesystem(temp_dir=tmp_path) as isolated_dir: + root_dir = pathlib.Path(isolated_dir) + + result = runner.invoke(main, args=['dot-env', root_dir.as_posix()], input='\n'.join(inputs)) + assert not result.exception + assert result.exit_code == 0 + paths = frozenset(map(lambda x: str(x.relative_to(isolated_dir).as_posix()), tmp_path.rglob('*'))) + assert '.env' in paths + + +def test_dot_env_contents(tmp_path: pathlib.Path): + runner = click.testing.CliRunner() + inputs = ['https://example.com', 'mySecretKey'] + with runner.isolated_filesystem(temp_dir=tmp_path) as isolated_dir: + root_dir = pathlib.Path(isolated_dir) + dot_env = root_dir / '.env' + + result = runner.invoke(main, args=['dot-env', root_dir.as_posix()], input='\n'.join(inputs)) + assert not result.exception + assert result.exit_code == 0 + paths = frozenset(map(lambda x: str(x.relative_to(isolated_dir).as_posix()), tmp_path.rglob('*'))) + assert '.env' in paths + contents = dot_env.read_text(encoding='utf-8') + assert 'DSW_API_URL=https://example.com' in contents + assert 'DSW_API_KEY=mySecretKey' in contents + + +def test_dot_env_no_force(tmp_path: pathlib.Path): + runner = click.testing.CliRunner() + inputs = ['https://example.com', 'mySecretKey'] + with runner.isolated_filesystem(temp_dir=tmp_path) as isolated_dir: + root_dir = pathlib.Path(isolated_dir) + dot_env = root_dir / '.env' + dot_env.write_text('test') + + result = runner.invoke(main, args=['dot-env', root_dir.as_posix()], input='\n'.join(inputs)) + assert result.exception + assert result.exit_code == 1 + + contents = dot_env.read_text(encoding='utf-8') + assert 'test' in contents + + +def test_dot_env_overwrites(tmp_path: pathlib.Path): + runner = click.testing.CliRunner() + inputs = ['https://example.com', 'mySecretKey'] + with runner.isolated_filesystem(temp_dir=tmp_path) as isolated_dir: + root_dir = pathlib.Path(isolated_dir) + dot_env = root_dir / '.env' + dot_env.write_text('test') + + result = runner.invoke(main, args=['dot-env', root_dir.as_posix(), '--force'], input='\n'.join(inputs)) + print(result.stdout) + assert not result.exception + assert result.exit_code == 0 + paths = frozenset(map(lambda x: str(x.relative_to(isolated_dir).as_posix()), tmp_path.rglob('*'))) + assert '.env' in paths + contents = dot_env.read_text(encoding='utf-8') + assert 'DSW_API_URL=https://example.com' in contents + assert 'DSW_API_KEY=mySecretKey' in contents diff --git a/packages/dsw-tdk/tests/test_cmd_get.py b/packages/dsw-tdk/tests/test_cmd_get.py index c049031b..d7dd562f 100644 --- a/packages/dsw-tdk/tests/test_cmd_get.py +++ b/packages/dsw-tdk/tests/test_cmd_get.py @@ -57,7 +57,7 @@ def test_get_draft_custom_dir(tmp_path: pathlib.Path, dsw_env: dict): @pytest.mark.vcr def test_put_bad_token(dsw_api_url: str): env_vars = { - 'DSW_API': dsw_api_url, + 'DSW_API_URL': dsw_api_url, 'DSW_API_KEY': 'foo', } runner = click.testing.CliRunner() @@ -68,7 +68,7 @@ def test_put_bad_token(dsw_api_url: str): @pytest.mark.vcr def test_put_bad_url(dsw_api_key: str): env_vars = { - 'DSW_API': 'http://localhost:33333', + 'DSW_API_URL': 'http://localhost:33333', 'DSW_API_KEY': dsw_api_key, } runner = click.testing.CliRunner() diff --git a/packages/dsw-tdk/tests/test_cmd_list.py b/packages/dsw-tdk/tests/test_cmd_list.py index d149afb4..dcc38e32 100644 --- a/packages/dsw-tdk/tests/test_cmd_list.py +++ b/packages/dsw-tdk/tests/test_cmd_list.py @@ -34,7 +34,7 @@ def test_list_released_only(dsw_env: dict): @pytest.mark.vcr def test_put_bad_token(dsw_api_url: str): env_vars = { - 'DSW_API': dsw_api_url, + 'DSW_API_URL': dsw_api_url, 'DSW_API_KEY': 'foo', } runner = click.testing.CliRunner() @@ -45,7 +45,7 @@ def test_put_bad_token(dsw_api_url: str): @pytest.mark.vcr def test_put_bad_url(dsw_api_key: str): env_vars = { - 'DSW_API': 'http://localhost:33333', + 'DSW_API_URL': 'http://localhost:33333', 'DSW_API_KEY': dsw_api_key, } runner = click.testing.CliRunner() diff --git a/packages/dsw-tdk/tests/test_cmd_put.py b/packages/dsw-tdk/tests/test_cmd_put.py index 2a886d17..8ba8b186 100644 --- a/packages/dsw-tdk/tests/test_cmd_put.py +++ b/packages/dsw-tdk/tests/test_cmd_put.py @@ -11,6 +11,7 @@ def test_put_ok(fixtures_path: pathlib.Path, dsw_env: dict): runner = click.testing.CliRunner() template_path = fixtures_path / 'test_example01' result = runner.invoke(main, args=['put', template_path.as_posix()], env=dsw_env) + print(result.stdout) assert result.exit_code == 0 @@ -35,7 +36,7 @@ def test_put_bad_token(fixtures_path: pathlib.Path, dsw_api_url: str): runner = click.testing.CliRunner() template_path = fixtures_path / 'test_example01' env_vars = { - 'DSW_API': dsw_api_url, + 'DSW_API_URL': dsw_api_url, 'DSW_API_KEY': 'foo', } result = runner.invoke(main, args=['put', template_path.as_posix()], env=env_vars) @@ -47,7 +48,7 @@ def test_put_bad_url(fixtures_path: pathlib.Path, dsw_api_key: str): runner = click.testing.CliRunner() template_path = fixtures_path / 'test_example01' env_vars = { - 'DSW_API': 'http://localhost:33333', + 'DSW_API_URL': 'http://localhost:33333', 'DSW_API_KEY': dsw_api_key, } result = runner.invoke(main, args=['put', template_path.as_posix()], env=env_vars) diff --git a/requirements.txt b/requirements.txt index c1ebacdc..7e2d56c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,9 @@ -Brotli==1.0.9 -Jinja2==3.1.2 -Markdown==3.4.4 -MarkupSafe==2.1.3 -Pillow==9.5.0 -PyYAML==6.0.1 -XlsxWriter==3.1.2 aiohttp==3.8.5 aiosignal==1.3.1 anyio==3.7.1 async-timeout==4.0.2 attrs==23.1.0 +Brotli==1.0.9 certifi==2023.7.22 cffi==1.15.1 charset-normalizer==3.2.0 @@ -17,21 +11,25 @@ click==8.1.6 colorama==0.4.6 cssselect2==0.7.0 dkimpy==1.1.5 -dnspython==2.4.1 -fonttools==4.41.1 +dnspython==2.4.2 +fonttools==4.42.0 frozenlist==1.4.0 html5lib==1.1 humanize==4.7.0 idna==3.4 isodate==0.6.1 +Jinja2==3.1.2 +Markdown==3.4.4 +MarkupSafe==2.1.3 mdx-breakless-lists==1.0.1 minio==7.1.15 multidict==6.0.4 pathspec==0.11.2 pathvalidate==3.1.0 pdfrw==0.4 -psycopg-binary==3.1.9 -psycopg==3.1.9 +Pillow==10.0.0 +psycopg==3.1.10 +psycopg-binary==3.1.10 pycparser==2.21 pydyf==0.7.0 pyparsing==3.1.1 @@ -39,10 +37,11 @@ pyphen==0.14.0 python-dateutil==2.8.2 python-dotenv==1.0.0 python-slugify==8.0.1 +PyYAML==6.0.1 +rdflib==7.0.0 rdflib-jsonld==0.6.2 -rdflib==6.3.2 requests==2.31.0 -sentry-sdk==1.28.1 +sentry-sdk==1.29.2 six==1.16.0 sniffio==1.3.0 tenacity==8.2.2 @@ -50,9 +49,10 @@ text-unidecode==1.3 tinycss2==1.2.1 typing_extensions==4.7.1 tzdata==2023.3 -urllib3==1.26.16 +urllib3==2.0.4 watchgod==0.8.2 weasyprint==59.0 webencodings==0.5.1 +XlsxWriter==3.1.2 yarl==1.9.2 zopfli==0.2.2 diff --git a/scripts/update-deps.py b/scripts/update-deps.py new file mode 100644 index 00000000..8379f618 --- /dev/null +++ b/scripts/update-deps.py @@ -0,0 +1,66 @@ +import pathlib +import shutil +import subprocess +import venv + +ENC = 'utf-8' + + +def run_update(root_dir: pathlib.Path): + _update_root_requirements(root_dir) + _update_packages(root_dir) + +def _update_root_requirements(root_dir: pathlib.Path): + env_dir = root_dir / 'env-deps' + pip = env_dir / 'bin' / 'pip' + pip_upgrade = env_dir / 'bin' / 'pip-upgrade' + requirements_file = root_dir / 'requirements.txt' + reqs = requirements_file.read_text(ENC) + reqs.replace('==', '>=') + requirements_file.write_text(reqs, ENC) + venv.create( + env_dir=env_dir, + system_site_packages=False, + clear=True, + with_pip=True, + ) + p = subprocess.Popen( + args=[pip, 'install', 'pip-upgrader'], + ) + p.wait() + p = subprocess.Popen( + args=[pip_upgrade, requirements_file.as_posix()], + ) + p.wait() + shutil.rmtree(env_dir) + +def _update_packages(root_dir: pathlib.Path): + requirements_file = root_dir / 'requirements.txt' + packages_dir = root_dir / 'packages' + deps = _load_deps(requirements_file) + for path in packages_dir.rglob('**/requirements.txt'): + _update_pkg_requirements(path, deps) + +def _update_pkg_requirements(requirements_file: pathlib.Path, deps: dict[str, str]): + lines = [] + for line in requirements_file.read_text(ENC).splitlines(): + parts = line.split('==') + if len(parts) != 2: + continue + dep = parts[0] + ver = deps.get(dep, parts[1]) + lines.append(f'{dep}=={ver}') + lines.append('') + requirements_file.write_text('\n'.join(lines), ENC) + +def _load_deps(requirements_file: pathlib.Path) -> dict[str, str]: + result = {} + for line in requirements_file.read_text(ENC).splitlines(): + parts = line.split('==') + if len(parts) == 2: + result[parts[0]] = parts[1] + return result + + +if __name__ == '__main__': + run_update(pathlib.Path(__file__).parent.parent)