From 40090b2925c5201af89e746c9c5068432dda68da Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Sat, 10 Feb 2024 16:05:19 +0100 Subject: [PATCH] churn: use async where possible --- otterdog/operations/delete_file.py | 6 +- otterdog/operations/diff_operation.py | 7 +- otterdog/operations/dispatch_workflow.py | 2 +- otterdog/operations/fetch_config.py | 16 +- otterdog/operations/import_configuration.py | 19 ++- otterdog/operations/list_apps.py | 2 +- otterdog/operations/list_members.py | 6 +- otterdog/operations/local_plan.py | 5 +- otterdog/operations/push_config.py | 16 +- otterdog/operations/show.py | 17 +- otterdog/operations/show_live.py | 2 +- otterdog/operations/sync_template.py | 6 +- otterdog/operations/validate.py | 4 +- otterdog/operations/web_login.py | 2 +- otterdog/providers/github/__init__.py | 10 +- otterdog/providers/github/graphql.py | 4 +- otterdog/providers/github/rest/__init__.py | 10 +- otterdog/providers/github/rest/app_client.py | 10 +- .../providers/github/rest/commit_client.py | 2 +- .../providers/github/rest/content_client.py | 6 +- .../providers/github/rest/issue_client.py | 2 +- otterdog/providers/github/rest/org_client.py | 82 +++++----- .../github/rest/pull_request_client.py | 4 +- otterdog/providers/github/rest/repo_client.py | 132 +++++++--------- otterdog/providers/github/rest/requester.py | 145 +++++------------- otterdog/providers/github/rest/team_client.py | 6 +- otterdog/providers/github/rest/user_client.py | 2 +- otterdog/webapp/utils.py | 18 ++- 28 files changed, 232 insertions(+), 311 deletions(-) diff --git a/otterdog/operations/delete_file.py b/otterdog/operations/delete_file.py index 1a11e655..2e82be71 100644 --- a/otterdog/operations/delete_file.py +++ b/otterdog/operations/delete_file.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* -import os +import aiofiles.ospath from otterdog.config import OrganizationConfig from otterdog.models.github_organization import GitHubOrganization @@ -53,7 +53,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: try: org_file_name = jsonnet_config.org_config_file - if not os.path.exists(org_file_name): + if not await aiofiles.ospath.exists(org_file_name): self.printer.print_error( f"configuration file '{org_file_name}' does not yet exist, run 'fetch-config' or 'import' first." ) @@ -71,7 +71,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.print_error(f"invalid credentials\n{str(e)}") return 1 - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: rest_api = provider.rest_api repositories_by_name = associate_by_key(organization.repositories, lambda r: r.name) diff --git a/otterdog/operations/diff_operation.py b/otterdog/operations/diff_operation.py index 735a4f17..db11b88b 100644 --- a/otterdog/operations/diff_operation.py +++ b/otterdog/operations/diff_operation.py @@ -8,10 +8,11 @@ from __future__ import annotations -import os from abc import abstractmethod from typing import Any, Optional, Protocol +import aiofiles.ospath + from otterdog.config import OrganizationConfig, OtterdogConfig from otterdog.jsonnet import JsonnetConfig from otterdog.models import LivePatch, LivePatchContext, LivePatchType, ModelObject @@ -88,7 +89,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: return await self._generate_diff(org_config) finally: self.printer.level_down() - self._gh_client.close() + await self._gh_client.close() def setup_github_client(self, org_config: OrganizationConfig) -> GitHubProvider: return GitHubProvider(self.config.get_credentials(org_config, only_token=self.no_web_ui)) @@ -115,7 +116,7 @@ async def _generate_diff(self, org_config: OrganizationConfig) -> int: org_file_name = jsonnet_config.org_config_file - if not os.path.exists(org_file_name): + if not await aiofiles.ospath.exists(org_file_name): self.printer.print_error( f"configuration file '{org_file_name}' does not yet exist, run fetch-config or import first." ) diff --git a/otterdog/operations/dispatch_workflow.py b/otterdog/operations/dispatch_workflow.py index aea49e65..ba169549 100644 --- a/otterdog/operations/dispatch_workflow.py +++ b/otterdog/operations/dispatch_workflow.py @@ -47,7 +47,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.print_error(f"invalid credentials\n{str(e)}") return 1 - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: success = await provider.dispatch_workflow(github_id, self.repo_name, self.workflow_name) if success is True: self.printer.println(f"workflow '{self.workflow_name}' dispatched for repo '{self.repo_name}'") diff --git a/otterdog/operations/fetch_config.py b/otterdog/operations/fetch_config.py index 1e3a671c..3cf746e4 100644 --- a/otterdog/operations/fetch_config.py +++ b/otterdog/operations/fetch_config.py @@ -6,7 +6,9 @@ # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* -import os +import aiofiles +import aiofiles.os +import aiofiles.ospath from otterdog.config import OrganizationConfig from otterdog.providers.github import GitHubProvider @@ -36,7 +38,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: org_file_name = jsonnet_config.org_config_file - if os.path.exists(org_file_name) and not self.force_processing: + if await aiofiles.ospath.exists(org_file_name) and not self.force_processing: self.printer.println() self.printer.println(style("Definition already exists", bright=True) + f" at '{org_file_name}'.") self.printer.println(" Performing this action will overwrite its contents.") @@ -56,7 +58,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.print_error(f"invalid credentials\n{str(e)}") return 1 - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: try: if self.pull_request is not None: ref = await provider.get_ref_for_pull_request( @@ -76,11 +78,11 @@ async def execute(self, org_config: OrganizationConfig) -> int: return 1 output_dir = jsonnet_config.org_dir - if not os.path.exists(output_dir): - os.makedirs(output_dir) + if not await aiofiles.ospath.exists(output_dir): + await aiofiles.os.makedirs(output_dir) - with open(org_file_name, "w") as file: - file.write(definition) + async with aiofiles.open(org_file_name, "w") as file: + await file.write(definition) if ref is not None: self.printer.println( diff --git a/otterdog/operations/import_configuration.py b/otterdog/operations/import_configuration.py index eaf9b262..fbb0bf62 100644 --- a/otterdog/operations/import_configuration.py +++ b/otterdog/operations/import_configuration.py @@ -6,9 +6,12 @@ # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* -import os import shutil +import aiofiles +import aiofiles.os +import aiofiles.ospath + from otterdog.config import OrganizationConfig from otterdog.models import PatchContext from otterdog.models.github_organization import GitHubOrganization @@ -48,7 +51,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: org_file_name = jsonnet_config.org_config_file - if os.path.exists(org_file_name) and not self.force_processing: + if await aiofiles.ospath.exists(org_file_name) and not self.force_processing: self.printer.println() self.printer.println(style("Definition already exists", bright=True) + f" at '{org_file_name}'.") self.printer.println(" Performing this action will overwrite its contents.") @@ -59,7 +62,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.println("\nImport cancelled.") return 1 - if os.path.exists(org_file_name): + if await aiofiles.ospath.exists(org_file_name): sync_secrets_from_previous_config = True backup_file = f"{org_file_name}.bak" shutil.copy(org_file_name, backup_file) @@ -82,7 +85,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: "the resulting config will be incomplete." ) - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: organization = await GitHubOrganization.load_from_provider( github_id, jsonnet_config, provider, self.no_web_ui, self.printer ) @@ -97,11 +100,11 @@ async def execute(self, org_config: OrganizationConfig) -> int: output = organization.to_jsonnet(jsonnet_config, context) output_dir = jsonnet_config.org_dir - if not os.path.exists(output_dir): - os.makedirs(output_dir) + if not await aiofiles.ospath.exists(output_dir): + await aiofiles.os.makedirs(output_dir) - with open(org_file_name, "w") as file: - file.write(output) + async with aiofiles.open(org_file_name, "w") as file: + await file.write(output) self.printer.println(f"Organization definition written to '{org_file_name}'.") diff --git a/otterdog/operations/list_apps.py b/otterdog/operations/list_apps.py index f235fcdc..17a21b4d 100644 --- a/otterdog/operations/list_apps.py +++ b/otterdog/operations/list_apps.py @@ -51,7 +51,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.print_error(f"invalid credentials\n{str(e)}") return 1 - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: apps = await provider.rest_api.org.get_app_installations(github_id) if not self.json_output: diff --git a/otterdog/operations/list_members.py b/otterdog/operations/list_members.py index ace45f28..792c6d3d 100644 --- a/otterdog/operations/list_members.py +++ b/otterdog/operations/list_members.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* -import os +import aiofiles.ospath from otterdog.config import OrganizationConfig from otterdog.models.github_organization import GitHubOrganization @@ -51,7 +51,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: try: org_file_name = jsonnet_config.org_config_file - if not os.path.exists(org_file_name): + if not await aiofiles.ospath.exists(org_file_name): self.printer.print_error( f"configuration file '{org_file_name}' does not yet exist, run fetch-config or import first" ) @@ -69,7 +69,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.print_error(f"invalid credentials\n{str(e)}") return 1 - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: members = await provider.rest_api.org.list_members(github_id, self.two_factor_disabled) if self.two_factor_disabled is True: diff --git a/otterdog/operations/local_plan.py b/otterdog/operations/local_plan.py index 77b19a86..a94612c5 100644 --- a/otterdog/operations/local_plan.py +++ b/otterdog/operations/local_plan.py @@ -6,9 +6,10 @@ # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* -import os from typing import Optional +import aiofiles.ospath + from otterdog.config import OrganizationConfig from otterdog.jsonnet import JsonnetConfig from otterdog.models.github_organization import GitHubOrganization @@ -47,7 +48,7 @@ def setup_github_client(self, org_config: OrganizationConfig) -> GitHubProvider: async def load_current_org(self, github_id: str, jsonnet_config: JsonnetConfig) -> GitHubOrganization: other_org_file_name = jsonnet_config.org_config_file + self.suffix - if not os.path.exists(other_org_file_name): + if not await aiofiles.ospath.exists(other_org_file_name): raise RuntimeError(f"configuration file '{other_org_file_name}' does not exist") return GitHubOrganization.load_from_file(github_id, other_org_file_name, self.config) diff --git a/otterdog/operations/push_config.py b/otterdog/operations/push_config.py index 28510241..108750b8 100644 --- a/otterdog/operations/push_config.py +++ b/otterdog/operations/push_config.py @@ -6,9 +6,11 @@ # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* -import os.path from typing import Optional +import aiofiles +import aiofiles.ospath + from otterdog.config import OrganizationConfig from otterdog.providers.github import GitHubProvider from otterdog.utils import style @@ -41,7 +43,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: org_file_name = jsonnet_config.org_config_file - if not os.path.exists(org_file_name): + if not await aiofiles.ospath.exists(org_file_name): self.printer.print_error( f"configuration file '{org_file_name}' does not yet exist, run fetch-config or import first" ) @@ -56,13 +58,13 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.print_error(f"invalid credentials\n{str(e)}") return 1 - with open(org_file_name, "r") as file: - content = file.read() + async with aiofiles.open(org_file_name, "r") as file: + content = await file.read() - with open(jsonnet_config.jsonnet_bundle_file, "r") as file: - bundle_content = file.read() + async with aiofiles.open(jsonnet_config.jsonnet_bundle_file, "r") as file: + bundle_content = await file.read() - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: try: updated_files = [] updated = False diff --git a/otterdog/operations/show.py b/otterdog/operations/show.py index 3ed9f858..c34e3ca2 100644 --- a/otterdog/operations/show.py +++ b/otterdog/operations/show.py @@ -10,6 +10,9 @@ import textwrap from io import StringIO +import aiofiles.os +import aiofiles.ospath + from otterdog.config import OrganizationConfig from otterdog.models import ModelObject from otterdog.models.github_organization import GitHubOrganization @@ -53,7 +56,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: try: org_file_name = jsonnet_config.org_config_file - if not os.path.exists(org_file_name): + if not await aiofiles.ospath.exists(org_file_name): self.printer.print_error( f"configuration file '{org_file_name}' does not yet exist, run 'fetch-config' or 'import first'" ) @@ -68,7 +71,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: if not self.markdown: self._print_classic(organization) else: - self._print_markdown(organization) + await self._print_markdown(organization) return 0 @@ -82,9 +85,9 @@ def _print_classic(self, organization: GitHubOrganization) -> None: model_header = model_object.get_model_header(parent_object) self.print_dict(model_object.to_model_dict(), model_header, "", "black") - def _print_markdown(self, organization: GitHubOrganization) -> None: - if not os.path.exists(self.output_dir): - os.makedirs(self.output_dir, exist_ok=True) + async def _print_markdown(self, organization: GitHubOrganization) -> None: + if not await aiofiles.ospath.exists(self.output_dir): + await aiofiles.os.makedirs(self.output_dir, exist_ok=True) writer = StringIO() self.printer = IndentingPrinter(writer, spaces_per_level=4) @@ -181,8 +184,8 @@ def _print_markdown(self, organization: GitHubOrganization) -> None: self.printer.level_down() - with open(os.path.join(self.output_dir, "configuration.md"), "w") as file: - file.write(writer.getvalue()) + async with aiofiles.open(os.path.join(self.output_dir, "configuration.md"), "w") as file: + await file.write(writer.getvalue()) for repo in organization.repositories: self._print_repo_markdown(organization, repo) diff --git a/otterdog/operations/show_live.py b/otterdog/operations/show_live.py index 8c990df5..7f7170f8 100644 --- a/otterdog/operations/show_live.py +++ b/otterdog/operations/show_live.py @@ -45,7 +45,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.print_error(f"invalid credentials\n{str(e)}") return 1 - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: if self.no_web_ui is True: self.printer.print_warn( "the Web UI will not be queried as '--no-web-ui' has been specified, " diff --git a/otterdog/operations/sync_template.py b/otterdog/operations/sync_template.py index 18b6ea8f..fbe38185 100644 --- a/otterdog/operations/sync_template.py +++ b/otterdog/operations/sync_template.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* -import os +import aiofiles.ospath from otterdog.config import OrganizationConfig from otterdog.models.github_organization import GitHubOrganization @@ -43,7 +43,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: try: org_file_name = jsonnet_config.org_config_file - if not os.path.exists(org_file_name): + if not await aiofiles.ospath.exists(org_file_name): self.printer.print_error( f"configuration file '{org_file_name}' does not yet exist, run fetch-config or import first." ) @@ -61,7 +61,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.print_error(f"invalid credentials\n{str(e)}") return 1 - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: rest_api = provider.rest_api repositories_by_name = associate_by_key(organization.repositories, lambda r: r.name) diff --git a/otterdog/operations/validate.py b/otterdog/operations/validate.py index 65ba2124..998d6f23 100644 --- a/otterdog/operations/validate.py +++ b/otterdog/operations/validate.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: EPL-2.0 # ******************************************************************************* -import os +import aiofiles.ospath from otterdog.config import OrganizationConfig from otterdog.models import FailureType @@ -38,7 +38,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: try: org_file_name = jsonnet_config.org_config_file - if not os.path.exists(org_file_name): + if not await aiofiles.ospath.exists(org_file_name): self.printer.print_error( f"configuration file '{org_file_name}' does not yet exist, run 'fetch-config' or 'import' first." ) diff --git a/otterdog/operations/web_login.py b/otterdog/operations/web_login.py index afdda828..8d928d70 100644 --- a/otterdog/operations/web_login.py +++ b/otterdog/operations/web_login.py @@ -37,7 +37,7 @@ async def execute(self, org_config: OrganizationConfig) -> int: self.printer.print_error(f"invalid credentials\n{str(e)}") return 1 - with GitHubProvider(credentials) as provider: + async with GitHubProvider(credentials) as provider: await provider.web_client.open_browser_with_logged_in_user(github_id) return 0 diff --git a/otterdog/providers/github/__init__.py b/otterdog/providers/github/__init__.py index 2c32e901..53da8982 100644 --- a/otterdog/providers/github/__init__.py +++ b/otterdog/providers/github/__init__.py @@ -42,15 +42,15 @@ def __init__(self, credentials: Optional[Credentials]): if credentials is not None: self._init_clients() - def __enter__(self): + async def __aenter__(self): return self - def __exit__(self, exception_type, exception_value, exception_traceback): - self.close() + async def __aexit__(self, exception_type, exception_value, exception_traceback): + await self.close() - def close(self) -> None: + async def close(self) -> None: if self._credentials is not None: - self.rest_api.close() + await self.rest_api.close() def _init_clients(self): self.rest_api = RestApi(token_auth(self._credentials.github_token)) diff --git a/otterdog/providers/github/graphql.py b/otterdog/providers/github/graphql.py index 768010bc..93fe215a 100644 --- a/otterdog/providers/github/graphql.py +++ b/otterdog/providers/github/graphql.py @@ -29,7 +29,7 @@ def __init__(self, auth_strategy: AuthStrategy): } async def get_branch_protection_rules(self, org_id: str, repo_name: str) -> list[dict[str, Any]]: - print_debug(f"async retrieving branch protection rules for repo '{org_id}/{repo_name}'") + print_debug(f"retrieving branch protection rules for repo '{org_id}/{repo_name}'") variables = {"organization": org_id, "repository": repo_name} branch_protection_rules = await self._async_run_paged_query(variables, "get-branch-protection-rules.gql") @@ -170,7 +170,7 @@ async def _async_run_paged_query( query_file: str, prefix_selector: str = ".data.repository.branchProtectionRules", ) -> list[dict[str, Any]]: - print_debug(f"running async graphql query '{query_file}' with input '{json.dumps(input_variables)}'") + print_debug(f"running graphql query '{query_file}' with input '{json.dumps(input_variables)}'") query = files(resources).joinpath(f"graphql/{query_file}").read_text() diff --git a/otterdog/providers/github/rest/__init__.py b/otterdog/providers/github/rest/__init__.py index 04298d13..23a9ffd4 100644 --- a/otterdog/providers/github/rest/__init__.py +++ b/otterdog/providers/github/rest/__init__.py @@ -27,6 +27,12 @@ def __init__(self, auth_strategy: Optional[AuthStrategy] = None): self._auth_strategy = auth_strategy self._requester = Requester(auth_strategy, self._GH_API_URL_ROOT, self._GH_API_VERSION) + async def __aenter__(self): + return self + + async def __aexit__(self, exception_type, exception_value, exception_traceback): + await self.close() + @property def token(self) -> Optional[str]: from otterdog.providers.github.auth.token import TokenAuthStrategy @@ -36,8 +42,8 @@ def token(self) -> Optional[str]: else: return None - def close(self) -> None: - self._requester.close() + async def close(self) -> None: + await self._requester.close() @property def requester(self) -> Requester: diff --git a/otterdog/providers/github/rest/app_client.py b/otterdog/providers/github/rest/app_client.py index 46bae8ac..b518e9b1 100644 --- a/otterdog/providers/github/rest/app_client.py +++ b/otterdog/providers/github/rest/app_client.py @@ -23,7 +23,7 @@ async def get_authenticated_app(self) -> dict[str, Any]: print_debug("retrieving authenticated app") try: - return await self.requester.async_request_json("GET", "/app") + return await self.requester.request_json("GET", "/app") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving authenticated app:\n{ex}").with_traceback(tb) @@ -32,7 +32,7 @@ async def get_app_installations(self) -> list[dict[str, Any]]: print_debug("retrieving app installations") try: - return await self.requester.async_request_paged_json("GET", "/app/installations") + return await self.requester.request_paged_json("GET", "/app/installations") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving authenticated app:\n{ex}").with_traceback(tb) @@ -41,9 +41,7 @@ async def create_installation_access_token(self, installation_id: str) -> tuple[ print_debug(f"creating an installation access token for installation '{installation_id}'") try: - response = await self.requester.async_request_json( - "POST", f"/app/installations/{installation_id}/access_tokens" - ) + response = await self.requester.request_json("POST", f"/app/installations/{installation_id}/access_tokens") return response["token"], parse_date_string(response["expires_at"]) except GitHubException as ex: tb = ex.__traceback__ @@ -53,7 +51,7 @@ async def get_app_ids(self, app_slug: str) -> tuple[int, str]: print_debug("retrieving app node id") try: - response = await self.requester.async_request_json("GET", f"/apps/{app_slug}") + response = await self.requester.request_json("GET", f"/apps/{app_slug}") return response["id"], response["node_id"] except GitHubException as ex: tb = ex.__traceback__ diff --git a/otterdog/providers/github/rest/commit_client.py b/otterdog/providers/github/rest/commit_client.py index da5c8c9a..3f7bd7e7 100644 --- a/otterdog/providers/github/rest/commit_client.py +++ b/otterdog/providers/github/rest/commit_client.py @@ -31,7 +31,7 @@ async def create_commit_status( print_debug(f"creating a commit status for sha '{sha}'") data = {"state": state, "target_url": target_url, "description": description, "context": context} - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "POST", f"/repos/{org_id}/{repo_name}/statuses/{sha}", data=json.dumps(data) ) diff --git a/otterdog/providers/github/rest/content_client.py b/otterdog/providers/github/rest/content_client.py index b2d2c6ae..878d1dcf 100644 --- a/otterdog/providers/github/rest/content_client.py +++ b/otterdog/providers/github/rest/content_client.py @@ -30,7 +30,7 @@ async def get_content_object( else: params = None - return await self.requester.async_request_json( + return await self.requester.request_json( "GET", f"/repos/{org_id}/{repo_name}/contents/{path}", params=params ) except GitHubException as ex: @@ -87,7 +87,7 @@ async def update_content( data["sha"] = old_sha try: - await self.requester.async_request_json("PUT", f"/repos/{org_id}/{repo_name}/contents/{path}", data) + await self.requester.request_json("PUT", f"/repos/{org_id}/{repo_name}/contents/{path}", data) return True except GitHubException as ex: tb = ex.__traceback__ @@ -119,7 +119,7 @@ async def delete_content( data = {"message": push_message, "sha": old_sha} try: - await self.requester.async_request_json("DELETE", f"/repos/{org_id}/{repo_name}/contents/{path}", data) + await self.requester.request_json("DELETE", f"/repos/{org_id}/{repo_name}/contents/{path}", data) return True except GitHubException as ex: tb = ex.__traceback__ diff --git a/otterdog/providers/github/rest/issue_client.py b/otterdog/providers/github/rest/issue_client.py index d23f2468..333e7053 100644 --- a/otterdog/providers/github/rest/issue_client.py +++ b/otterdog/providers/github/rest/issue_client.py @@ -21,7 +21,7 @@ async def create_comment(self, org_id: str, repo_name: str, issue_number: str, b try: data = {"body": body} - return await self.requester.async_request_json( + return await self.requester.request_json( "POST", f"/repos/{org_id}/{repo_name}/issues/{issue_number}/comments", data=data ) except GitHubException as ex: diff --git a/otterdog/providers/github/rest/org_client.py b/otterdog/providers/github/rest/org_client.py index c3747849..5ddf820a 100644 --- a/otterdog/providers/github/rest/org_client.py +++ b/otterdog/providers/github/rest/org_client.py @@ -24,7 +24,7 @@ async def get_settings(self, org_id: str, included_keys: set[str]) -> dict[str, print_debug(f"retrieving settings for org '{org_id}'") try: - settings = await self.requester.async_request_json("GET", f"/orgs/{org_id}") + settings = await self.requester.request_json("GET", f"/orgs/{org_id}") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving settings for organization '{org_id}':\n{ex}").with_traceback(tb) @@ -45,7 +45,7 @@ async def update_settings(self, org_id: str, data: dict[str, Any]) -> None: print_debug(f"updating settings for organization '{org_id}'") try: - await self.requester.async_request_json("PATCH", f"/orgs/{org_id}", data) + await self.requester.request_json("PATCH", f"/orgs/{org_id}", data) except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed to update settings for organization '{org_id}':\n{ex}").with_traceback(tb) @@ -59,7 +59,7 @@ async def list_security_managers(self, org_id: str) -> list[str]: print_debug(f"retrieving security managers for organization {org_id}") try: - result = await self.requester.async_request_json("GET", f"/orgs/{org_id}/security-managers") + result = await self.requester.request_json("GET", f"/orgs/{org_id}/security-managers") return list(map(lambda x: x["slug"], result)) except GitHubException as ex: tb = ex.__traceback__ @@ -86,9 +86,7 @@ async def update_security_managers(self, org_id: str, security_managers: list[st async def add_security_manager_team(self, org_id: str, team_slug: str) -> None: print_debug(f"adding team {team_slug} to security managers for organization {org_id}") - status, body = await self.requester.async_request_raw( - "PUT", f"/orgs/{org_id}/security-managers/teams/{team_slug}" - ) + status, body = await self.requester.request_raw("PUT", f"/orgs/{org_id}/security-managers/teams/{team_slug}") if status == 204: print_debug(f"added team {team_slug} to security managers for organization {org_id}") @@ -105,9 +103,7 @@ async def add_security_manager_team(self, org_id: str, team_slug: str) -> None: async def remove_security_manager_team(self, org_id: str, team_slug: str) -> None: print_debug(f"removing team {team_slug} from security managers for organization {org_id}") - status, body = await self.requester.async_request_raw( - "DELETE", f"/orgs/{org_id}/security-managers/teams/{team_slug}" - ) + status, body = await self.requester.request_raw("DELETE", f"/orgs/{org_id}/security-managers/teams/{team_slug}") if status != 204: raise RuntimeError( f"failed removing team '{team_slug}' from security managers of organization '{org_id}'" @@ -120,7 +116,7 @@ async def get_webhooks(self, org_id: str) -> list[dict[str, Any]]: print_debug(f"retrieving org webhooks for org '{org_id}'") try: - return await self.requester.async_request_json("GET", f"/orgs/{org_id}/hooks") + return await self.requester.request_json("GET", f"/orgs/{org_id}/hooks") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving webhooks for org '{org_id}':\n{ex}").with_traceback(tb) @@ -129,7 +125,7 @@ async def update_webhook(self, org_id: str, webhook_id: int, webhook: dict[str, print_debug(f"updating org webhook '{webhook_id}' for organization {org_id}") try: - await self.requester.async_request_json("PATCH", f"/orgs/{org_id}/hooks/{webhook_id}", webhook) + await self.requester.request_json("PATCH", f"/orgs/{org_id}/hooks/{webhook_id}", webhook) print_debug(f"updated webhook {webhook_id}") except GitHubException as ex: tb = ex.__traceback__ @@ -143,7 +139,7 @@ async def add_webhook(self, org_id: str, data: dict[str, Any]) -> None: data["name"] = "web" try: - await self.requester.async_request_json("POST", f"/orgs/{org_id}/hooks", data) + await self.requester.request_json("POST", f"/orgs/{org_id}/hooks", data) print_debug(f"added org webhook with url '{url}'") except GitHubException as ex: tb = ex.__traceback__ @@ -152,7 +148,7 @@ async def add_webhook(self, org_id: str, data: dict[str, Any]) -> None: async def delete_webhook(self, org_id: str, webhook_id: int, url: str) -> None: print_debug(f"deleting org webhook with url '{url}'") - status, _ = await self.requester.async_request_raw("DELETE", f"/orgs/{org_id}/hooks/{webhook_id}") + status, _ = await self.requester.request_raw("DELETE", f"/orgs/{org_id}/hooks/{webhook_id}") if status != 204: raise RuntimeError(f"failed to delete org webhook with url '{url}'") @@ -164,7 +160,7 @@ async def get_repos(self, org_id: str) -> list[str]: params = {"type": "all"} try: - repos = await self.requester.async_request_paged_json("GET", f"/orgs/{org_id}/repos", params) + repos = await self.requester.request_paged_json("GET", f"/orgs/{org_id}/repos", params) return [repo["name"] for repo in repos] except GitHubException as ex: tb = ex.__traceback__ @@ -174,7 +170,7 @@ async def get_secrets(self, org_id: str) -> list[dict[str, Any]]: print_debug(f"retrieving secrets for org '{org_id}'") try: - response = await self.requester.async_request_json("GET", f"/orgs/{org_id}/actions/secrets") + response = await self.requester.request_json("GET", f"/orgs/{org_id}/actions/secrets") secrets = response["secrets"] for secret in secrets: @@ -192,7 +188,7 @@ async def _get_selected_repositories_for_secret(self, org_id: str, secret_name: try: url = f"/orgs/{org_id}/actions/secrets/{secret_name}/repositories" - response = await self.requester.async_request_json("GET", url) + response = await self.requester.request_json("GET", url) return response["repositories"] except GitHubException as ex: tb = ex.__traceback__ @@ -207,7 +203,7 @@ async def _set_selected_repositories_for_secret( data = {"selected_repository_ids": selected_repository_ids} url = f"/orgs/{org_id}/actions/secrets/{secret_name}/repositories" - status, _ = await self.requester.async_request_raw("PUT", url, json.dumps(data)) + status, _ = await self.requester.request_raw("PUT", url, json.dumps(data)) if status != 204: raise RuntimeError(f"failed to update selected repositories for secret '{secret_name}'") else: @@ -235,7 +231,7 @@ async def update_secret(self, org_id: str, secret_name: str, secret: dict[str, A await self._encrypt_secret_inplace(org_id, secret) - status, _ = await self.requester.async_request_raw( + status, _ = await self.requester.request_raw( "PUT", f"/orgs/{org_id}/actions/secrets/{secret_name}", json.dumps(secret) ) @@ -253,7 +249,7 @@ async def add_secret(self, org_id: str, data: dict[str, str]) -> None: await self._encrypt_secret_inplace(org_id, data) - status, _ = await self.requester.async_request_raw( + status, _ = await self.requester.request_raw( "PUT", f"/orgs/{org_id}/actions/secrets/{secret_name}", json.dumps(data) ) @@ -272,7 +268,7 @@ async def _encrypt_secret_inplace(self, org_id: str, data: dict[str, Any]) -> No async def delete_secret(self, org_id: str, secret_name: str) -> None: print_debug(f"deleting org secret '{secret_name}'") - status, _ = await self.requester.async_request_raw("DELETE", f"/orgs/{org_id}/actions/secrets/{secret_name}") + status, _ = await self.requester.request_raw("DELETE", f"/orgs/{org_id}/actions/secrets/{secret_name}") if status != 204: raise RuntimeError(f"failed to delete org secret '{secret_name}'") @@ -282,7 +278,7 @@ async def get_variables(self, org_id: str) -> list[dict[str, Any]]: print_debug(f"retrieving variables for org '{org_id}'") try: - response = await self.requester.async_request_json("GET", f"/orgs/{org_id}/actions/variables") + response = await self.requester.request_json("GET", f"/orgs/{org_id}/actions/variables") secrets = response["variables"] for secret in secrets: @@ -300,7 +296,7 @@ async def _get_selected_repositories_for_variable(self, org_id: str, variable_na try: url = f"/orgs/{org_id}/actions/variables/{variable_name}/repositories" - response = await self.requester.async_request_json("GET", url) + response = await self.requester.request_json("GET", url) return response["repositories"] except GitHubException as ex: tb = ex.__traceback__ @@ -315,7 +311,7 @@ async def _set_selected_repositories_for_variable( data = {"selected_repository_ids": selected_repository_ids} url = f"/orgs/{org_id}/actions/variables/{variable_name}/repositories" - status, _ = await self.requester.async_request_raw("PUT", url, json.dumps(data)) + status, _ = await self.requester.request_raw("PUT", url, json.dumps(data)) if status != 204: raise RuntimeError(f"failed to update selected repositories for variable '{variable_name}'") @@ -341,7 +337,7 @@ async def update_variable(self, org_id: str, variable_name: str, variable: dict[ else: selected_repository_ids = None - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "PATCH", f"/orgs/{org_id}/actions/variables/{variable_name}", json.dumps(variable) ) if status != 204: @@ -356,9 +352,7 @@ async def add_variable(self, org_id: str, data: dict[str, str]) -> None: variable_name = data.get("name") print_debug(f"adding org variable '{variable_name}'") - status, body = await self.requester.async_request_raw( - "POST", f"/orgs/{org_id}/actions/variables", json.dumps(data) - ) + status, body = await self.requester.request_raw("POST", f"/orgs/{org_id}/actions/variables", json.dumps(data)) if status != 201: raise RuntimeError(f"failed to add org variable '{variable_name}': {body}") @@ -368,9 +362,7 @@ async def add_variable(self, org_id: str, data: dict[str, str]) -> None: async def delete_variable(self, org_id: str, variable_name: str) -> None: print_debug(f"deleting org variable '{variable_name}'") - status, body = await self.requester.async_request_raw( - "DELETE", f"/orgs/{org_id}/actions/variables/{variable_name}" - ) + status, body = await self.requester.request_raw("DELETE", f"/orgs/{org_id}/actions/variables/{variable_name}") if status != 204: raise RuntimeError(f"failed to delete org variable '{variable_name}': {body}") @@ -381,7 +373,7 @@ async def get_public_key(self, org_id: str) -> tuple[str, str]: print_debug(f"retrieving org public key for org '{org_id}'") try: - response = await self.requester.async_request_json("GET", f"/orgs/{org_id}/actions/secrets/public-key") + response = await self.requester.request_json("GET", f"/orgs/{org_id}/actions/secrets/public-key") return response["key_id"], response["key"] except GitHubException as ex: tb = ex.__traceback__ @@ -392,7 +384,7 @@ async def get_team_ids(self, combined_slug: str) -> tuple[int, str]: org_id, team_slug = re.split("/", combined_slug) try: - response = await self.requester.async_request_json("GET", f"/orgs/{org_id}/teams/{team_slug}") + response = await self.requester.request_json("GET", f"/orgs/{org_id}/teams/{team_slug}") return response["id"], response["node_id"] except GitHubException as ex: tb = ex.__traceback__ @@ -402,7 +394,7 @@ async def get_teams(self, org_id: str) -> list[dict[str, Any]]: print_debug(f"retrieving teams for org '{org_id}'") try: - return await self.requester.async_request_json("GET", f"/orgs/{org_id}/teams") + return await self.requester.request_json("GET", f"/orgs/{org_id}/teams") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving teams for org '{org_id}':\n{ex}").with_traceback(tb) @@ -411,7 +403,7 @@ async def get_app_installations(self, org_id: str) -> list[dict[str, Any]]: print_debug(f"retrieving app installations for org '{org_id}'") try: - response = await self.requester.async_request_json("GET", f"/orgs/{org_id}/installations") + response = await self.requester.request_json("GET", f"/orgs/{org_id}/installations") return response["installations"] except GitHubException as ex: tb = ex.__traceback__ @@ -423,7 +415,7 @@ async def get_workflow_settings(self, org_id: str) -> dict[str, Any]: workflow_settings: dict[str, Any] = {} try: - permissions = await self.requester.async_request_json("GET", f"/orgs/{org_id}/actions/permissions") + permissions = await self.requester.request_json("GET", f"/orgs/{org_id}/actions/permissions") workflow_settings.update(permissions) except GitHubException as ex: tb = ex.__traceback__ @@ -450,7 +442,7 @@ async def update_workflow_settings(self, org_id: str, data: dict[str, Any]) -> N permission_data = {k: data[k] for k in ["enabled_repositories", "allowed_actions"] if k in data} if len(permission_data) > 0: - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "PUT", f"/orgs/{org_id}/actions/permissions", json.dumps(permission_data) ) @@ -480,9 +472,7 @@ async def _get_selected_repositories_for_workflow_settings(self, org_id: str) -> print_debug("retrieving selected repositories for org workflow settings") try: - response = await self.requester.async_request_json( - "GET", f"/orgs/{org_id}/actions/permissions/repositories" - ) + response = await self.requester.request_json("GET", f"/orgs/{org_id}/actions/permissions/repositories") return response["repositories"] except GitHubException as ex: tb = ex.__traceback__ @@ -494,7 +484,7 @@ async def _update_selected_repositories_for_workflow_settings( print_debug("updating selected repositories for org workflow settings") data = {"selected_repository_ids": selected_repository_ids} - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "PUT", f"/orgs/{org_id}/actions/permissions/repositories", json.dumps(data) ) @@ -509,9 +499,7 @@ async def _get_selected_actions_for_workflow_settings(self, org_id: str) -> dict print_debug(f"retrieving allowed actions for org '{org_id}'") try: - return await self.requester.async_request_json( - "GET", f"/orgs/{org_id}/actions/permissions/selected-actions" - ) + return await self.requester.request_json("GET", f"/orgs/{org_id}/actions/permissions/selected-actions") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving allowed actions for org '{org_id}':\n{ex}").with_traceback(tb) @@ -519,7 +507,7 @@ async def _get_selected_actions_for_workflow_settings(self, org_id: str) -> dict async def _update_selected_actions_for_workflow_settings(self, org_id: str, data: dict[str, Any]) -> None: print_debug(f"updating allowed actions for org '{org_id}'") - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "PUT", f"/orgs/{org_id}/actions/permissions/selected-actions", json.dumps(data) ) @@ -532,7 +520,7 @@ async def _get_default_workflow_permissions(self, org_id: str) -> dict[str, Any] print_debug(f"retrieving default workflow permissions for org '{org_id}'") try: - return await self.requester.async_request_json("GET", f"/orgs/{org_id}/actions/permissions/workflow") + return await self.requester.request_json("GET", f"/orgs/{org_id}/actions/permissions/workflow") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving org default workflow permissions:\n{ex}").with_traceback(tb) @@ -540,7 +528,7 @@ async def _get_default_workflow_permissions(self, org_id: str) -> dict[str, Any] async def _update_default_workflow_permissions(self, org_id: str, data: dict[str, Any]) -> None: print_debug(f"updating default workflow permissions for org '{org_id}'") - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "PUT", f"/orgs/{org_id}/actions/permissions/workflow", json.dumps(data) ) @@ -554,7 +542,7 @@ async def list_members(self, org_id: str, two_factor_disabled: bool) -> list[dic try: params = "?filter=2fa_disabled" if two_factor_disabled is True else "" - return await self.requester.async_request_paged_json("GET", f"/orgs/{org_id}/members{params}") + return await self.requester.request_paged_json("GET", f"/orgs/{org_id}/members{params}") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving org default workflow permissions:\n{ex}").with_traceback(tb) diff --git a/otterdog/providers/github/rest/pull_request_client.py b/otterdog/providers/github/rest/pull_request_client.py index e358fd6e..cd1465df 100644 --- a/otterdog/providers/github/rest/pull_request_client.py +++ b/otterdog/providers/github/rest/pull_request_client.py @@ -22,9 +22,7 @@ async def get_pull_request(self, org_id: str, repo_name: str, pull_request_numbe print_debug(f"getting pull request with number '{pull_request_number}' from repo '{org_id}/{repo_name}'") try: - return await self.requester.async_request_json( - "GET", f"/repos/{org_id}/{repo_name}/pulls/{pull_request_number}" - ) + return await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/pulls/{pull_request_number}") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving pull request:\n{ex}").with_traceback(tb) diff --git a/otterdog/providers/github/rest/repo_client.py b/otterdog/providers/github/rest/repo_client.py index c1560e3c..55a65efb 100644 --- a/otterdog/providers/github/rest/repo_client.py +++ b/otterdog/providers/github/rest/repo_client.py @@ -12,8 +12,9 @@ import re import tempfile import zipfile -from typing import IO, Any, Optional +from typing import Any, Optional +import aiofiles import chevron import jq # type: ignore @@ -35,7 +36,7 @@ async def get_repo_data(self, org_id: str, repo_name: str) -> dict[str, Any]: print_debug(f"retrieving org repo data for '{org_id}/{repo_name}'") try: - repo_data = await self.requester.async_request_json("GET", f"/repos/{org_id}/{repo_name}") + repo_data = await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}") await self._fill_github_pages_config(org_id, repo_name, repo_data) await self._fill_vulnerability_report(org_id, repo_name, repo_data) await self._fill_topics(org_id, repo_name, repo_data) @@ -48,7 +49,7 @@ async def get_repo_by_id(self, repo_id: int) -> dict[str, Any]: print_debug(f"retrieving repo by id for '{repo_id}'") try: - return await self.requester.async_request_json("GET", f"/repositories/{repo_id}") + return await self.requester.request_json("GET", f"/repositories/{repo_id}") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving data for repo '{repo_id}':\n{ex}").with_traceback(tb) @@ -81,7 +82,7 @@ async def update_repo(self, org_id: str, repo_name: str, data: dict[str, Any]) - if changes > 0: try: if len(data) > 0: - await self.requester.async_request_json("PATCH", f"/repos/{org_id}/{repo_name}", data) + await self.requester.request_json("PATCH", f"/repos/{org_id}/{repo_name}", data) if vulnerability_reports is not None: await self._update_vulnerability_report(org_id, repo_name, vulnerability_reports) @@ -120,7 +121,7 @@ async def add_repo( "default_branch_only": fork_default_branch_only, } - await self.requester.async_request_json( + await self.requester.request_json( "POST", f"/repos/{upstream_owner}/{upstream_repo}/forks", fork_data, @@ -152,7 +153,7 @@ async def add_repo( "private": data.get("private", False), } - await self.requester.async_request_json( + await self.requester.request_json( "POST", f"/repos/{template_owner}/{template_repo}/generate", template_data, @@ -226,7 +227,7 @@ async def add_repo( data["auto_init"] = auto_init_repo try: - result = await self.requester.async_request_json("POST", f"/orgs/{org_id}/repos", data) + result = await self.requester.request_json("POST", f"/orgs/{org_id}/repos", data) print_debug(f"created repo with name '{repo_name}'") self._remove_already_active_settings(update_data, result) await self.update_repo(org_id, repo_name, update_data) @@ -238,7 +239,7 @@ async def get_webhooks(self, org_id: str, repo_name: str) -> list[dict[str, Any] print_debug(f"retrieving webhooks for repo '{org_id}/{repo_name}'") try: - return await self.requester.async_request_json("GET", f"/repos/{org_id}/{repo_name}/hooks") + return await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/hooks") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed retrieving webhooks for repo '{org_id}/{repo_name}':\n{ex}").with_traceback(tb) @@ -247,7 +248,7 @@ async def update_webhook(self, org_id: str, repo_name: str, webhook_id: int, web print_debug(f"updating repo webhook '{webhook_id}' for repo '{org_id}/{repo_name}'") try: - await self.requester.async_request_json("PATCH", f"/repos/{org_id}/{repo_name}/hooks/{webhook_id}", webhook) + await self.requester.request_json("PATCH", f"/repos/{org_id}/{repo_name}/hooks/{webhook_id}", webhook) print_debug(f"updated repo webhook '{webhook_id}'") except GitHubException as ex: tb = ex.__traceback__ @@ -261,7 +262,7 @@ async def add_webhook(self, org_id: str, repo_name: str, data: dict[str, Any]) - data["name"] = "web" try: - await self.requester.async_request_json("POST", f"/repos/{org_id}/{repo_name}/hooks", data) + await self.requester.request_json("POST", f"/repos/{org_id}/{repo_name}/hooks", data) print_debug(f"added repo webhook with url '{url}'") except GitHubException as ex: tb = ex.__traceback__ @@ -270,7 +271,7 @@ async def add_webhook(self, org_id: str, repo_name: str, data: dict[str, Any]) - async def delete_webhook(self, org_id: str, repo_name: str, webhook_id: int, url: str) -> None: print_debug(f"deleting repo webhook with url '{url}' for repo '{org_id}/{repo_name}'") - status, _ = await self.requester.async_request_raw("DELETE", f"/repos/{org_id}/{repo_name}/hooks/{webhook_id}") + status, _ = await self.requester.request_raw("DELETE", f"/repos/{org_id}/{repo_name}/hooks/{webhook_id}") if status != 204: raise RuntimeError(f"failed to delete repo webhook with url '{url}'") @@ -283,7 +284,7 @@ async def get_rulesets(self, org_id: str, repo_name: str) -> list[dict[str, Any] try: result = [] params = {"includes_parents": str(False)} - response = await self.requester.async_request_paged_json( + response = await self.requester.request_paged_json( "GET", f"/repos/{org_id}/{repo_name}/rulesets", params=params ) for ruleset in response: @@ -298,7 +299,7 @@ async def get_ruleset(self, org_id: str, repo_name: str, ruleset_id: str) -> dic try: params = {"includes_parents": str(False)} - return await self.requester.async_request_json( + return await self.requester.request_json( "GET", f"/repos/{org_id}/{repo_name}/rulesets/{ruleset_id}", params=params ) except GitHubException as ex: @@ -309,9 +310,7 @@ async def update_ruleset(self, org_id: str, repo_name: str, ruleset_id: int, rul print_debug(f"updating repo ruleset '{ruleset_id}' for repo '{org_id}/{repo_name}'") try: - await self.requester.async_request_json( - "PUT", f"/repos/{org_id}/{repo_name}/rulesets/{ruleset_id}", ruleset - ) + await self.requester.request_json("PUT", f"/repos/{org_id}/{repo_name}/rulesets/{ruleset_id}", ruleset) print_debug(f"updated repo ruleset '{ruleset_id}'") except GitHubException as ex: tb = ex.__traceback__ @@ -325,7 +324,7 @@ async def add_ruleset(self, org_id: str, repo_name: str, data: dict[str, Any]) - data["target"] = "branch" try: - await self.requester.async_request_json("POST", f"/repos/{org_id}/{repo_name}/rulesets", data) + await self.requester.request_json("POST", f"/repos/{org_id}/{repo_name}/rulesets", data) print_debug(f"added repo ruleset with name '{name}'") except GitHubException as ex: tb = ex.__traceback__ @@ -334,9 +333,7 @@ async def add_ruleset(self, org_id: str, repo_name: str, data: dict[str, Any]) - async def delete_ruleset(self, org_id: str, repo_name: str, ruleset_id: int, name: str) -> None: print_debug(f"deleting repo ruleset with name '{name}' for repo '{org_id}/{repo_name}'") - status, _ = await self.requester.async_request_raw( - "DELETE", f"/repos/{org_id}/{repo_name}/rulesets/{ruleset_id}" - ) + status, _ = await self.requester.request_raw("DELETE", f"/repos/{org_id}/{repo_name}/rulesets/{ruleset_id}") if status != 204: raise RuntimeError(f"failed to delete repo ruleset with name '{name}'") @@ -352,7 +349,7 @@ async def get_readme(self, org_id: str, repo_name: str) -> dict[str, Any]: print_debug(f"getting readme for repo '{org_id}/{repo_name}'") try: - return await self.requester.async_request_json("GET", f"/repos/{org_id}/{repo_name}/readme") + return await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/readme") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed to get readme for repo '{org_id}/{repo_name}':\n{ex}").with_traceback(tb) @@ -360,7 +357,7 @@ async def get_readme(self, org_id: str, repo_name: str) -> dict[str, Any]: async def delete_repo(self, org_id: str, repo_name: str) -> None: print_debug(f"deleting repo '{org_id}/{repo_name}'") - status, body = await self.requester.async_request_raw("DELETE", f"/repos/{org_id}/{repo_name}") + status, body = await self.requester.request_raw("DELETE", f"/repos/{org_id}/{repo_name}") if status != 204: raise RuntimeError(f"failed to delete repo '{org_id}/{repo_name}': {body}") @@ -382,7 +379,7 @@ def _remove_already_active_settings(update_data: dict[str, Any], current_data: d async def _fill_github_pages_config(self, org_id: str, repo_name: str, repo_data: dict[str, Any]) -> None: print_debug(f"retrieving github pages config for '{org_id}/{repo_name}'") - status, body = await self.requester.async_request_raw("GET", f"/repos/{org_id}/{repo_name}/pages") + status, body = await self.requester.request_raw("GET", f"/repos/{org_id}/{repo_name}/pages") if status == 200: repo_data["gh_pages"] = json.loads(body) @@ -417,11 +414,11 @@ async def _update_github_pages_config(self, org_id: str, repo_name: str, gh_page build_type = gh_pages.get("build_type", None) if build_type == "disabled": - self.requester.request_raw("DELETE", f"/repos/{org_id}/{repo_name}/pages") + await self.requester.request_raw("DELETE", f"/repos/{org_id}/{repo_name}/pages") else: gh_pages_data: list[tuple[str, str, int]] = [] # first check if the pages config already exists: - status, _ = await self.requester.async_request_raw("GET", f"/repos/{org_id}/{repo_name}/pages") + status, _ = await self.requester.request_raw("GET", f"/repos/{org_id}/{repo_name}/pages") if status != 200: # check if the branch already exists source: Any = gh_pages.get("source", None) @@ -446,9 +443,7 @@ async def _update_github_pages_config(self, org_id: str, repo_name: str, gh_page gh_pages_data.append((json.dumps(gh_pages), "PUT", 204)) for data, method, status_code in gh_pages_data: - status, body = await self.requester.async_request_raw( - method, f"/repos/{org_id}/{repo_name}/pages", data=data - ) + status, body = await self.requester.request_raw(method, f"/repos/{org_id}/{repo_name}/pages", data=data) if status != status_code: raise RuntimeError(f"failed to update github pages config for repo '{repo_name}': {body}") @@ -467,13 +462,13 @@ async def _update_default_branch(self, org_id: str, repo_name: str, new_default_ try: if new_default_branch in existing_branch_names: data = {"default_branch": new_default_branch} - await self.requester.async_request_json("PATCH", f"/repos/{org_id}/{repo_name}", data) + await self.requester.request_json("PATCH", f"/repos/{org_id}/{repo_name}", data) print_debug(f"updated default branch for '{org_id}/{repo_name}'") else: repo = await self.get_repo_data(org_id, repo_name) default_branch = repo["default_branch"] data = {"new_name": new_default_branch} - await self.requester.async_request_json( + await self.requester.request_json( "POST", f"/repos/{org_id}/{repo_name}/branches/{default_branch}/rename", data ) print_debug(f"renamed default branch for '{org_id}/{repo_name}'") @@ -486,7 +481,7 @@ async def _update_default_branch(self, org_id: str, repo_name: str, new_default_ async def _fill_vulnerability_report(self, org_id: str, repo_name: str, repo_data: dict[str, Any]) -> None: print_debug(f"retrieving repo vulnerability report status for '{org_id}/{repo_name}'") - status, _ = await self.requester.async_request_raw("GET", f"/repos/{org_id}/{repo_name}/vulnerability-alerts") + status, _ = await self.requester.request_raw("GET", f"/repos/{org_id}/{repo_name}/vulnerability-alerts") if status == 204: repo_data["dependabot_alerts_enabled"] = True else: @@ -500,9 +495,7 @@ async def _update_vulnerability_report(self, org_id: str, repo_name: str, vulner else: method = "DELETE" - status, body = await self.requester.async_request_raw( - method, f"/repos/{org_id}/{repo_name}/vulnerability-alerts" - ) + status, body = await self.requester.request_raw(method, f"/repos/{org_id}/{repo_name}/vulnerability-alerts") if status != 204: raise RuntimeError(f"failed to update vulnerability_reports for repo '{repo_name}': {body}") @@ -513,7 +506,7 @@ async def _fill_topics(self, org_id: str, repo_name: str, repo_data: dict[str, A print_debug(f"retrieving repo topics for '{org_id}/{repo_name}'") try: - response = await self.requester.async_request_json("GET", f"/repos/{org_id}/{repo_name}/topics") + response = await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/topics") repo_data["topics"] = response.get("names", []) except GitHubException as ex: tb = ex.__traceback__ @@ -522,14 +515,14 @@ async def _fill_topics(self, org_id: str, repo_name: str, repo_data: dict[str, A async def _update_topics(self, org_id: str, repo_name: str, topics: list[str]) -> None: print_debug(f"updating repo topics for '{org_id}/{repo_name}'") data = {"names": topics} - await self.requester.async_request_json("PUT", f"/repos/{org_id}/{repo_name}/topics", data=data) + await self.requester.request_json("PUT", f"/repos/{org_id}/{repo_name}/topics", data=data) print_debug(f"updated topics for repo '{repo_name}'") async def get_branches(self, org_id: str, repo_name) -> list[dict[str, Any]]: print_debug(f"retrieving branches for repo '{org_id}/{repo_name}'") try: - return await self.requester.async_request_json("GET", f"/repos/{org_id}/{repo_name}/branches") + return await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/branches") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError(f"failed getting branches for repo '{org_id}/{repo_name}':\n{ex}").with_traceback(tb) @@ -538,7 +531,7 @@ async def get_environments(self, org_id: str, repo_name: str) -> list[dict[str, print_debug(f"retrieving environments for repo '{org_id}/{repo_name}'") try: - response = await self.requester.async_request_json("GET", f"/repos/{org_id}/{repo_name}/environments") + response = await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/environments") environments = response["environments"] for env in environments: @@ -566,7 +559,7 @@ async def update_environment(self, org_id: str, repo_name: str, env_name: str, e branch_policies = None try: - await self.requester.async_request_json("PUT", f"/repos/{org_id}/{repo_name}/environments/{env_name}", env) + await self.requester.request_json("PUT", f"/repos/{org_id}/{repo_name}/environments/{env_name}", env) if branch_policies is not None: await self._update_deployment_branch_policies(org_id, repo_name, env_name, branch_policies) @@ -584,9 +577,7 @@ async def add_environment(self, org_id: str, repo_name: str, env_name: str, data async def delete_environment(self, org_id: str, repo_name: str, env_name: str) -> None: print_debug(f"deleting repo environment '{env_name} for repo '{org_id}/{repo_name}'") - status, _ = await self.requester.async_request_raw( - "DELETE", f"/repos/{org_id}/{repo_name}/environments/{env_name}" - ) + status, _ = await self.requester.request_raw("DELETE", f"/repos/{org_id}/{repo_name}/environments/{env_name}") if status != 204: raise RuntimeError(f"failed to delete repo environment '{env_name}'") @@ -598,7 +589,7 @@ async def _get_deployment_branch_policies(self, org_id: str, repo_name: str, env try: url = f"/repos/{org_id}/{repo_name}/environments/{env_name}/deployment-branch-policies" - response = await self.requester.async_request_json("GET", url) + response = await self.requester.request_json("GET", url) return response["branch_policies"] except GitHubException as ex: tb = ex.__traceback__ @@ -639,7 +630,7 @@ async def _create_deployment_branch_policy(self, org_id: str, repo_name: str, en try: data = {"name": name} url = f"/repos/{org_id}/{repo_name}/environments/{env_name}/deployment-branch-policies" - await self.requester.async_request_json("POST", url, data) + await self.requester.request_json("POST", url, data) print_debug(f"created deployment branch policy for env '{env_name}'") except GitHubException as ex: tb = ex.__traceback__ @@ -651,7 +642,7 @@ async def _delete_deployment_branch_policy( print_debug(f"deleting deployment branch policy for env '{env_name}' with id '{policy_id}") url = f"/repos/{org_id}/{repo_name}/environments/{env_name}/deployment-branch-policies/{policy_id}" - status, body = await self.requester.async_request_raw("DELETE", url) + status, body = await self.requester.request_raw("DELETE", url) if status != 204: raise RuntimeError(f"failed deleting deployment branch policy" f"\n{status}: {body}") @@ -662,7 +653,7 @@ async def get_secrets(self, org_id: str, repo_name: str) -> list[dict[str, Any]] print_debug(f"retrieving secrets for repo '{org_id}/{repo_name}'") try: - status, body = await self.requester.async_request_raw("GET", f"/repos/{org_id}/{repo_name}/actions/secrets") + status, body = await self.requester.request_raw("GET", f"/repos/{org_id}/{repo_name}/actions/secrets") if status == 200: return json.loads(body)["secrets"] else: @@ -679,7 +670,7 @@ async def update_secret(self, org_id: str, repo_name: str, secret_name: str, sec await self._encrypt_secret_inplace(org_id, repo_name, secret) - status, _ = await self.requester.async_request_raw( + status, _ = await self.requester.request_raw( "PUT", f"/repos/{org_id}/{repo_name}/actions/secrets/{secret_name}", json.dumps(secret), @@ -696,7 +687,7 @@ async def add_secret(self, org_id: str, repo_name: str, data: dict[str, str]) -> await self._encrypt_secret_inplace(org_id, repo_name, data) - status, _ = await self.requester.async_request_raw( + status, _ = await self.requester.request_raw( "PUT", f"/repos/{org_id}/{repo_name}/actions/secrets/{secret_name}", json.dumps(data), @@ -716,7 +707,7 @@ async def _encrypt_secret_inplace(self, org_id: str, repo_name: str, data: dict[ async def delete_secret(self, org_id: str, repo_name: str, secret_name: str) -> None: print_debug(f"deleting repo secret '{secret_name}' for repo '{org_id}/{repo_name}'") - status, _ = await self.requester.async_request_raw( + status, _ = await self.requester.request_raw( "DELETE", f"/repos/{org_id}/{repo_name}/actions/secrets/{secret_name}" ) @@ -729,9 +720,7 @@ async def get_variables(self, org_id: str, repo_name: str) -> list[dict[str, Any print_debug(f"retrieving variables for repo '{org_id}/{repo_name}'") try: - status, body = await self.requester.async_request_raw( - "GET", f"/repos/{org_id}/{repo_name}/actions/variables" - ) + status, body = await self.requester.request_raw("GET", f"/repos/{org_id}/{repo_name}/actions/variables") if status == 200: return json.loads(body)["variables"] else: @@ -746,7 +735,7 @@ async def update_variable(self, org_id: str, repo_name: str, variable_name: str, if "name" in variable: variable.pop("name") - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "PATCH", f"/repos/{org_id}/{repo_name}/actions/variables/{variable_name}", json.dumps(variable), @@ -760,7 +749,7 @@ async def add_variable(self, org_id: str, repo_name: str, data: dict[str, str]) variable_name = data.get("name") print_debug(f"adding repo variable '{variable_name}' for repo '{org_id}/{repo_name}'") - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "POST", f"/repos/{org_id}/{repo_name}/actions/variables", json.dumps(data), @@ -774,7 +763,7 @@ async def add_variable(self, org_id: str, repo_name: str, data: dict[str, str]) async def delete_variable(self, org_id: str, repo_name: str, variable_name: str) -> None: print_debug(f"deleting repo variable '{variable_name}' for repo '{org_id}/{repo_name}'") - status, _ = await self.requester.async_request_raw( + status, _ = await self.requester.request_raw( "DELETE", f"/repos/{org_id}/{repo_name}/actions/variables/{variable_name}" ) @@ -789,9 +778,7 @@ async def get_workflow_settings(self, org_id: str, repo_name: str) -> dict[str, workflow_settings: dict[str, Any] = {} try: - permissions = await self.requester.async_request_json( - "GET", f"/repos/{org_id}/{repo_name}/actions/permissions" - ) + permissions = await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/actions/permissions") workflow_settings.update(permissions) except GitHubException as ex: tb = ex.__traceback__ @@ -813,7 +800,7 @@ async def update_workflow_settings(self, org_id: str, repo_name: str, data: dict permission_data = {k: data[k] for k in ["enabled", "allowed_actions"] if k in data} if len(permission_data) > 0: - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "PUT", f"/repos/{org_id}/{repo_name}/actions/permissions", json.dumps(permission_data) ) @@ -844,7 +831,7 @@ async def _get_selected_actions_for_workflow_settings(self, org_id: str, repo_na print_debug(f"retrieving allowed actions for org '{org_id}'") try: - return await self.requester.async_request_json( + return await self.requester.request_json( "GET", f"/repos/{org_id}/{repo_name}/actions/permissions/selected-actions" ) except GitHubException as ex: @@ -858,7 +845,7 @@ async def _update_selected_actions_for_workflow_settings( ) -> None: print_debug(f"updating allowed actions for repo '{org_id}/{repo_name}'") - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "PUT", f"/repos/{org_id}/{repo_name}/actions/permissions/selected-actions", json.dumps(data) ) @@ -871,9 +858,7 @@ async def _get_default_workflow_permissions(self, org_id: str, repo_name: str) - print_debug(f"async retrieving default workflow permissions for repo '{org_id}/{repo_name}'") try: - return await self.requester.async_request_json( - "GET", f"/repos/{org_id}/{repo_name}/actions/permissions/workflow" - ) + return await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/actions/permissions/workflow") except GitHubException as ex: tb = ex.__traceback__ raise RuntimeError( @@ -883,7 +868,7 @@ async def _get_default_workflow_permissions(self, org_id: str, repo_name: str) - async def _update_default_workflow_permissions(self, org_id: str, repo_name: str, data: dict[str, Any]) -> None: print_debug(f"updating default workflow permissions for repo '{org_id}/{repo_name}'") - status, body = await self.requester.async_request_raw( + status, body = await self.requester.request_raw( "PUT", f"/repos/{org_id}/{repo_name}/actions/permissions/workflow", json.dumps(data) ) @@ -898,7 +883,7 @@ async def get_public_key(self, org_id: str, repo_name: str) -> tuple[str, str]: print_debug(f"retrieving repo public key for repo '{org_id}/{repo_name}'") try: - response = await self.requester.async_request_json( + response = await self.requester.request_json( "GET", f"/repos/{org_id}/{repo_name}/actions/secrets/public-key" ) return response["key_id"], response["key"] @@ -912,7 +897,7 @@ async def dispatch_workflow(self, org_id: str, repo_name: str, workflow_name: st repo_data = await self.get_repo_data(org_id, repo_name) data = {"ref": repo_data["default_branch"]} - status, _ = await self.requester.async_request_raw( + status, _ = await self.requester.request_raw( "POST", f"/repos/{org_id}/{repo_name}/actions/workflows/{workflow_name}/dispatches", json.dumps(data) ) @@ -927,9 +912,7 @@ async def get_ref_for_pull_request(self, org_id: str, repo_name: str, pull_numbe print_debug(f"retrieving ref for pull request {pull_number} at {org_id}/{repo_name}") try: - response = await self.requester.async_request_json( - "GET", f"/repos/{org_id}/{repo_name}/pulls/{pull_number}" - ) + response = await self.requester.request_json("GET", f"/repos/{org_id}/{repo_name}/pulls/{pull_number}") return response["head"]["sha"] except GitHubException as ex: tb = ex.__traceback__ @@ -947,7 +930,7 @@ async def sync_from_template_repository( updated_files = [] with tempfile.TemporaryDirectory() as tmp_dir: archive_file_name = os.path.join(tmp_dir, "archive.zip") - with open(archive_file_name, "wb") as archive_file: + async with aiofiles.open(archive_file_name, "wb") as archive_file: await self._download_repository_archive(archive_file, template_owner, template_repo) archive_target_dir = os.path.join(tmp_dir, "contents") @@ -983,13 +966,12 @@ async def sync_from_template_repository( return updated_files - async def _download_repository_archive(self, file: IO, org_id: str, repo_name: str, ref: str = "") -> None: + async def _download_repository_archive(self, file, org_id: str, repo_name: str, ref: str = "") -> None: print_debug(f"downloading repository archive for '{org_id}/{repo_name}'") try: - # TODO: use async streaming - with self.requester.request_raw("GET", f"/repos/{org_id}/{repo_name}/zipball/{ref}") as response: - file.write(response.content) + async for data in self.requester.request_stream("GET", f"/repos/{org_id}/{repo_name}/zipball/{ref}"): + await file.write(data) except GitHubException as ex: tb = ex.__traceback__ diff --git a/otterdog/providers/github/rest/requester.py b/otterdog/providers/github/rest/requester.py index 8dd8092f..86e60f9c 100644 --- a/otterdog/providers/github/rest/requester.py +++ b/otterdog/providers/github/rest/requester.py @@ -7,19 +7,16 @@ # ******************************************************************************* import json -from typing import Any, Optional +from typing import Any, AsyncIterable, Optional from aiohttp_client_cache.backends import FileBackend from aiohttp_client_cache.session import CachedSession as AsyncCachedSession -from requests import Response -from requests_cache import CachedSession from otterdog.providers.github.auth import AuthStrategy from otterdog.providers.github.exception import BadCredentialsException, GitHubException from otterdog.utils import is_debug_enabled, is_trace_enabled, print_debug, print_trace _AIOHTTP_CACHE_DIR = ".cache/async_http" -_REQUESTS_CACHE_DIR = ".cache/http" class Requester: @@ -33,28 +30,20 @@ def __init__(self, auth_strategy: Optional[AuthStrategy], base_url: str, api_ver "X-Github-Next-Global-ID": "1", } - # enable logging for requests_cache - # import logging - # - # - # logging.basicConfig(level='DEBUG') - - self._session: CachedSession = CachedSession( - _REQUESTS_CACHE_DIR, - backend="filesystem", - use_cache_dir=False, - cache_control=True, - allowable_methods=["GET"], + self._async_session = AsyncCachedSession( + cache=FileBackend( + cache_name=_AIOHTTP_CACHE_DIR, + use_temp=False, + ), ) - self._session.auth = self._auth - def close(self) -> None: - self._session.close() + async def close(self) -> None: + await self._async_session.close() def _build_url(self, url_path: str) -> str: return f"{self._base_url}{url_path}" - def request_paged_json( + async def request_paged_json( self, method: str, url_path: str, @@ -68,7 +57,7 @@ def request_paged_json( if params is not None: query_params.update(params) - response: list[dict[str, Any]] = self.request_json(method, url_path, data, query_params) + response: list[dict[str, Any]] = await self.request_json(method, url_path, data, query_params) if len(response) == 0: current_page = -1 @@ -80,48 +69,7 @@ def request_paged_json( return result - async def async_request_paged_json( - self, - method: str, - url_path: str, - data: Optional[dict[str, Any]] = None, - params: Optional[dict[str, str]] = None, - ) -> list[dict[str, Any]]: - result = [] - current_page = 1 - while current_page > 0: - query_params = {"per_page": "100", "page": current_page} - if params is not None: - query_params.update(params) - - response: list[dict[str, Any]] = await self.async_request_json(method, url_path, data, query_params) - - if len(response) == 0: - current_page = -1 - else: - for item in response: - result.append(item) - - current_page += 1 - - return result - - def request_json( - self, - method: str, - url_path: str, - data: Optional[dict[str, Any]] = None, - params: Optional[dict[str, Any]] = None, - ) -> Any: - input_data = None - if data is not None: - input_data = json.dumps(data) - - response = self.request_raw(method, url_path, input_data, params) - self._check_response(response.url, response.status_code, response.text) - return response.json() - - async def async_request_json( + async def request_json( self, method: str, url_path: str, @@ -132,71 +80,60 @@ async def async_request_json( if data is not None: input_data = json.dumps(data) - status, body = await self.async_request_raw(method, url_path, input_data, params) + status, body = await self.request_raw(method, url_path, input_data, params) self._check_response(url_path, status, body) return json.loads(body) - def request_raw( + async def request_raw( self, method: str, url_path: str, data: Optional[str] = None, params: Optional[dict[str, Any]] = None, - stream: bool = False, - force_refresh: bool = False, - ) -> Response: + ) -> tuple[int, str]: print_trace(f"'{method}' url = {url_path}, data = {data}, headers = {self._headers}") - response = self._session.request( - method, - url=self._build_url(url_path), - headers=self._headers, - refresh=True, - force_refresh=force_refresh, - params=params, - data=data, - stream=stream, - ) + headers = self._headers.copy() + if self._auth is not None: + self._auth.update_headers_with_authorization(headers) - if is_debug_enabled(): - print_debug(f"'{method}' {url_path}: rate-limit-used = {response.headers.get('x-ratelimit-used', None)}") + url = self._build_url(url_path) + async with self._async_session.request( + method, url=url, headers=headers, params=params, data=data, refresh=True + ) as response: + text = await response.text() + status = response.status - if is_trace_enabled(): - print_trace(f"'{method}' result = ({response.status_code}, {response.text})") + if is_debug_enabled(): + if not response.from_cache: # type: ignore + print_debug( + f"'{method}' {url_path}: rate-limit-used = {response.headers.get('x-ratelimit-used', None)}" + ) - return response + if is_trace_enabled(): + print_trace(f"async '{method}' result = ({status}, {text})") - async def async_request_raw( + return status, text + + async def request_stream( self, method: str, url_path: str, data: Optional[str] = None, params: Optional[dict[str, Any]] = None, - ) -> tuple[int, str]: - print_trace(f"async '{method}' url = {url_path}, data = {data}, headers = {self._headers}") + ) -> AsyncIterable[bytes]: + print_trace(f"stream '{method}' url = {url_path}, data = {data}, headers = {self._headers}") headers = self._headers.copy() if self._auth is not None: self._auth.update_headers_with_authorization(headers) - async with AsyncCachedSession(cache=FileBackend(cache_name=_AIOHTTP_CACHE_DIR, use_temp=False)) as session: - url = self._build_url(url_path) - async with session.request( - method, url=url, headers=headers, params=params, data=data, refresh=True - ) as response: - text = await response.text() - status = response.status - - if is_debug_enabled(): - if not response.from_cache: # type: ignore - print_debug( - f"'{method}' {url_path}: rate-limit-used = {response.headers.get('x-ratelimit-used', None)}" - ) - - if is_trace_enabled(): - print_trace(f"async '{method}' result = ({status}, {text})") - - return status, text + url = self._build_url(url_path) + async with self._async_session.request( + method, url=url, headers=headers, params=params, data=data, refresh=True + ) as response: + async for chunk, _ in response.content.iter_chunks(): + yield chunk def _check_response(self, url_path: str, status_code: int, body: str) -> None: if status_code >= 400: diff --git a/otterdog/providers/github/rest/team_client.py b/otterdog/providers/github/rest/team_client.py index 6ebc22a4..72cbddc3 100644 --- a/otterdog/providers/github/rest/team_client.py +++ b/otterdog/providers/github/rest/team_client.py @@ -21,7 +21,7 @@ async def get_team_slugs(self, org_id: str) -> list[dict[str, Any]]: print_debug(f"retrieving teams for org '{org_id}'") try: - response = await self.requester.async_request_json("GET", f"/orgs/{org_id}/teams") + response = await self.requester.request_json("GET", f"/orgs/{org_id}/teams") return list(map(lambda team: team["slug"], response)) except GitHubException as ex: tb = ex.__traceback__ @@ -30,9 +30,7 @@ async def get_team_slugs(self, org_id: str) -> list[dict[str, Any]]: async def is_user_member_of_team(self, org_id: str, team_slug: str, user: str) -> bool: print_debug(f"retrieving membership of user '{user}' for team '{team_slug}' in org '{org_id}'") - status, body = await self.requester.async_request_raw( - "GET", f"/orgs/{org_id}/teams/{team_slug}/memberships/{user}" - ) + status, body = await self.requester.request_raw("GET", f"/orgs/{org_id}/teams/{team_slug}/memberships/{user}") if status == 200: return True diff --git a/otterdog/providers/github/rest/user_client.py b/otterdog/providers/github/rest/user_client.py index 14e25cd5..3b3e5666 100644 --- a/otterdog/providers/github/rest/user_client.py +++ b/otterdog/providers/github/rest/user_client.py @@ -20,7 +20,7 @@ async def get_user_ids(self, login: str) -> tuple[int, str]: print_debug(f"retrieving user ids for user '{login}'") try: - response = await self.requester.async_request_json("GET", f"/users/{login}") + response = await self.requester.request_json("GET", f"/users/{login}") return response["id"], response["node_id"] except GitHubException as ex: tb = ex.__traceback__ diff --git a/otterdog/webapp/utils.py b/otterdog/webapp/utils.py index 97678bf1..b29f0f08 100644 --- a/otterdog/webapp/utils.py +++ b/otterdog/webapp/utils.py @@ -51,6 +51,8 @@ async def get_rest_api_for_installation(installation_id: int) -> RestApi: if current_api is not None and expires_at is not None: if expires_at > datetime.now(): return current_api + else: + await current_api.close() token, expires_at = await get_rest_api_for_app().app.create_installation_access_token(installation) rest_api = RestApi(token_auth(token)) @@ -83,15 +85,15 @@ async def _load_otterdog_config(ref: Optional[str] = None) -> OtterdogConfig: f"'https://github.com/{config_file_owner}/{config_file_repo}/{config_file_path}'" ) - rest_api = RestApi(token_auth(current_app.config["OTTERDOG_CONFIG_TOKEN"])) - content = await rest_api.content.get_content(config_file_owner, config_file_repo, config_file_path, ref) - import aiofiles + async with RestApi(token_auth(current_app.config["OTTERDOG_CONFIG_TOKEN"])) as rest_api: + content = await rest_api.content.get_content(config_file_owner, config_file_repo, config_file_path, ref) + import aiofiles - async with aiofiles.tempfile.NamedTemporaryFile("wt") as file: - name = cast(str, file.name) - await file.write(content) - await file.flush() - return OtterdogConfig(name, False, app_root) + async with aiofiles.tempfile.NamedTemporaryFile("wt") as file: + name = cast(str, file.name) + await file.write(content) + await file.flush() + return OtterdogConfig(name, False, app_root) async def get_organization_config(org_model: InstallationModel, token: str, work_dir: str) -> OrganizationConfig: