From 8c3d82a80a123bafb02888156b70b8629e789836 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Wed, 7 Feb 2024 11:39:05 +0100 Subject: [PATCH] Add replicate feature fixes #216 --- CHANGES/216.feature | 1 + pulp_gem/app/replica.py | 41 ++++++++++ .../functional/api/test_crud_content_unit.py | 21 ++--- .../functional/api/test_download_content.py | 9 ++- pulp_gem/tests/functional/api/test_publish.py | 17 +++-- .../tests/functional/api/test_replicate.py | 76 +++++++++++++++++++ pulp_gem/tests/functional/conftest.py | 12 ++- requirements.txt | 1 + 8 files changed, 147 insertions(+), 31 deletions(-) create mode 100644 CHANGES/216.feature create mode 100644 pulp_gem/app/replica.py create mode 100644 pulp_gem/tests/functional/api/test_replicate.py diff --git a/CHANGES/216.feature b/CHANGES/216.feature new file mode 100644 index 00000000..8b24b3fc --- /dev/null +++ b/CHANGES/216.feature @@ -0,0 +1 @@ +Added replica definitions. diff --git a/pulp_gem/app/replica.py b/pulp_gem/app/replica.py new file mode 100644 index 00000000..3d91dba5 --- /dev/null +++ b/pulp_gem/app/replica.py @@ -0,0 +1,41 @@ +from pulpcore.plugin.replica import Replicator + +from pulp_glue.gem.context import ( + PulpGemDistributionContext, + PulpGemPublicationContext, + PulpGemRepositoryContext, +) + +from pulp_gem.app.models import GemDistribution, GemRemote, GemRepository +from pulp_gem.app.tasks import synchronize as gem_synchronize + + +class GemReplicator(Replicator): + repository_ctx_cls = PulpGemRepositoryContext + distribution_ctx_cls = PulpGemDistributionContext + publication_ctx_cls = PulpGemPublicationContext + app_label = "gem" + remote_model_cls = GemRemote + repository_model_cls = GemRepository + distribution_model_cls = GemDistribution + distribution_serializer_name = "GemDistributionSerializer" + repository_serializer_name = "GemRepositorySerializer" + remote_serializer_name = "GemRemoteSerializer" + sync_task = gem_synchronize + + def url(self, upstream_distribution): + if upstream_distribution["publication"]: + return upstream_distribution["base_url"] + else: + # This distribution doesn't serve any content + return None + + def sync_params(self, repository, remote): + return dict( + remote_pk=str(remote.pk), + repository_pk=str(repository.pk), + mirror=True, + ) + + +REPLICATION_ORDER = [GemReplicator] diff --git a/pulp_gem/tests/functional/api/test_crud_content_unit.py b/pulp_gem/tests/functional/api/test_crud_content_unit.py index 237f1e2d..0fb9ae4f 100644 --- a/pulp_gem/tests/functional/api/test_crud_content_unit.py +++ b/pulp_gem/tests/functional/api/test_crud_content_unit.py @@ -1,21 +1,14 @@ """Tests that perform actions over content unit.""" -import uuid - import pytest -from pulp_gem.tests.functional.constants import GEM_URL def test_upload_content_unit( gem_content_api_client, - http_get, + gem_content_artifact, monitor_task, - tmp_path, delete_orphans_pre, ): - temp_file = tmp_path / str(uuid.uuid4()) - content = http_get(GEM_URL) - temp_file.write_bytes(content) - response = gem_content_api_client.create(file=temp_file) + response = gem_content_api_client.create(file=gem_content_artifact) task = monitor_task(response.task) assert len(task.created_resources) == 1 @@ -24,7 +17,7 @@ def test_upload_content_unit( assert content.version == "1.0.0" # Upload again - response = gem_content_api_client.create(file=temp_file) + response = gem_content_api_client.create(file=gem_content_artifact) task = monitor_task(response.task) assert len(task.created_resources) == 1 assert task.created_resources[0] == content.pulp_href @@ -32,18 +25,14 @@ def test_upload_content_unit( def test_crud_content_unit( gem_content_api_client, + gem_content_artifact, artifacts_api_client, - http_get, monitor_task, - tmp_path, delete_orphans_pre, ): """CRUD content unit.""" # 01 test create - temp_file = tmp_path / str(uuid.uuid4()) - content = http_get(GEM_URL) - temp_file.write_bytes(content) - artifact = artifacts_api_client.create(temp_file) + artifact = artifacts_api_client.create(gem_content_artifact) response = gem_content_api_client.create(artifact=artifact.pulp_href) task = monitor_task(response.task) assert len(task.created_resources) == 1 diff --git a/pulp_gem/tests/functional/api/test_download_content.py b/pulp_gem/tests/functional/api/test_download_content.py index e6f29ea2..d256a8ee 100644 --- a/pulp_gem/tests/functional/api/test_download_content.py +++ b/pulp_gem/tests/functional/api/test_download_content.py @@ -10,9 +10,9 @@ @pytest.mark.parametrize("policy", DOWNLOAD_POLICIES) def test_download_content( policy, - gem_repo, gem_repository_api_client, gem_content_api_client, + gem_repository_factory, gem_remote_factory, gem_publication_factory, gem_distribution_factory, @@ -30,16 +30,17 @@ def test_download_content( content unit from Pulp, and verify that the content unit has the same checksum when fetched directly from Pulp-Fixtures. """ + repository = gem_repository_factory() remote = gem_remote_factory(policy=policy) # Sync repository body = {"remote": remote.pulp_href} - result = gem_repository_api_client.sync(gem_repo.pulp_href, body) + result = gem_repository_api_client.sync(repository.pulp_href, body) monitor_task(result.task) - repo = gem_repository_api_client.read(gem_repo.pulp_href) + repo = gem_repository_api_client.read(repository.pulp_href) # Create a publication. - publication = gem_publication_factory(repository=gem_repo.pulp_href) + publication = gem_publication_factory(repository=repository.pulp_href) # Create a distribution. distribution = gem_distribution_factory(publication=publication.pulp_href) diff --git a/pulp_gem/tests/functional/api/test_publish.py b/pulp_gem/tests/functional/api/test_publish.py index 75bc09c1..83cbc27f 100644 --- a/pulp_gem/tests/functional/api/test_publish.py +++ b/pulp_gem/tests/functional/api/test_publish.py @@ -5,12 +5,12 @@ @pytest.mark.parallel def test_publish( - gem_repo, gem_repository_api_client, gem_repository_version_api_client, - gem_remote_factory, gem_content_api_client, gem_publication_factory, + gem_remote_factory, + gem_repository_factory, monitor_task, ): """Test whether a particular repository version can be published. @@ -23,10 +23,11 @@ def test_publish( 5. Assert that the publication ``repository_version`` attribute points to the supplied repository version. """ + repository = gem_repository_factory() remote = gem_remote_factory() - result = gem_repository_api_client.sync(gem_repo.pulp_href, {"remote": remote.pulp_href}) + result = gem_repository_api_client.sync(repository.pulp_href, {"remote": remote.pulp_href}) monitor_task(result.task) - repo = gem_repository_api_client.read(gem_repo.pulp_href) + repo = gem_repository_api_client.read(repository.pulp_href) # Step 1 content = gem_content_api_client.list(repository_version=repo.latest_version_href) @@ -36,16 +37,16 @@ def test_publish( result = gem_repository_api_client.modify(repo.pulp_href, body) monitor_task(result.task) - repo = gem_repository_api_client.read(repo.pulp_href) - assert repo.latest_version_href.endswith(f"/{content.count + 1}/") + repository = gem_repository_api_client.read(repository) + assert repository.latest_version_href.endswith(f"/{content.count + 1}/") - versions = gem_repository_version_api_client.list(repo.pulp_href) + versions = gem_repository_version_api_client.list(repository.pulp_href) # This is in descending order, so latest first version_hrefs = [ver.pulp_href for ver in versions.results] non_latest = choice(version_hrefs[:-1]) # Step 2 - publication = gem_publication_factory(repository=repo.pulp_href) + publication = gem_publication_factory(repository=repository.pulp_href) # Step 3 assert publication.repository_version == version_hrefs[0] diff --git a/pulp_gem/tests/functional/api/test_replicate.py b/pulp_gem/tests/functional/api/test_replicate.py new file mode 100644 index 00000000..580c515e --- /dev/null +++ b/pulp_gem/tests/functional/api/test_replicate.py @@ -0,0 +1,76 @@ +import pytest +import uuid + + +@pytest.mark.parallel +def test_replication( + domain_factory, + bindings_cfg, + pulpcore_bindings, + monitor_task_group, + pulp_settings, + add_to_cleanup, + gen_object_with_cleanup, + gem_distribution_factory, + gem_publication_factory, + gem_repository_factory, + gem_distribution_api_client, + gem_remote_api_client, + gem_repository_api_client, + gem_content_api_client, + gem_content_artifact, +): + # This test assures that an Upstream Pulp can be created in a non-default domain and that this + # Upstream Pulp configuration can be used to execute the replicate task. + + # Create a domain to replicate from + source_domain = domain_factory() + + # Create a domain as replica + replica_domain = domain_factory() + + # Add stuff to it + repository = gem_repository_factory(pulp_domain=source_domain.name) + gem_content_api_client.create( + file=gem_content_artifact, repository=repository, pulp_domain=source_domain.name + ) + publication = gem_publication_factory( + pulp_domain=source_domain.name, repository=repository.pulp_href + ) + gem_distribution_factory( + pulp_domain=source_domain.name, publication=publication.pulp_href + ) + + # Create an Upstream Pulp in the non-default domain + upstream_pulp_body = { + "name": str(uuid.uuid4()), + "base_url": bindings_cfg.host, + "api_root": pulp_settings.API_ROOT, + "domain": source_domain.name, + "username": bindings_cfg.username, + "password": bindings_cfg.password, + } + upstream_pulp = gen_object_with_cleanup( + pulpcore_bindings.UpstreamPulpsApi, upstream_pulp_body, pulp_domain=replica_domain.name + ) + # Run the replicate task and assert that all tasks successfully complete. + response = pulpcore_bindings.UpstreamPulpsApi.replicate(upstream_pulp.pulp_href) + monitor_task_group(response.task_group) + + for api_client in ( + gem_distribution_api_client, + gem_remote_api_client, + gem_repository_api_client, + ): + result = api_client.list(pulp_domain=replica_domain.name) + for item in result.results: + add_to_cleanup(api_client, item) + + for api_client in ( + gem_distribution_api_client, + gem_remote_api_client, + gem_repository_api_client, + gem_content_api_client, + ): + result = api_client.list(pulp_domain=replica_domain.name) + assert result.count == 1 diff --git a/pulp_gem/tests/functional/conftest.py b/pulp_gem/tests/functional/conftest.py index f6c3c18e..073601b3 100644 --- a/pulp_gem/tests/functional/conftest.py +++ b/pulp_gem/tests/functional/conftest.py @@ -11,6 +11,7 @@ RepositoriesGemApi, RepositoriesGemVersionsApi, ) +from pulp_gem.tests.functional.constants import GEM_URL # Api Bindings fixtures @@ -116,6 +117,11 @@ def _gem_remote_factory(*, url=GEM_FIXTURE_URL, policy="immediate", **kwargs): return _gem_remote_factory -@pytest.fixture -def gem_repo(gem_repository_factory): - return gem_repository_factory() +@pytest.fixture(scope="session") +def gem_content_artifact(tmp_path_factory, http_get): + """A file containing amber-1.0.0.gem.""" + + temp_file = tmp_path_factory.mktemp("pulp_gem") / "amber-1.0.0.gem" + content = http_get(GEM_URL) + temp_file.write_bytes(content) + return temp_file diff --git a/requirements.txt b/requirements.txt index 3da159e0..2434f30a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pulpcore>=3.39.0,<3.54 +pulp_cli_gem>=0.3.0,<0.4.0 rubymarshal>=1.2.7,<1.3