Add pull-through caching support

fixes: #94

.github/workflows/scripts/post_before_script.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# Install gem client before running the test
+SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "stream" "generate-bindings" "lowerbounds")
+if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
+  cmd_prefix dnf install -yq gem
+fi

CHANGES/94.feature

@@ -0,0 +1 @@
+Added support for pull-through caching. Add a remote to a distribution to enable this feature.

pulp_gem/app/models.py

@@ -1,15 +1,20 @@
 from logging import getLogger
 
-from django.db import models
+from django.db import models, utils
+from pathlib import PurePath
+from tempfile import NamedTemporaryFile
 
 from pulpcore.plugin.models import (
+    Artifact,
     Content,
     Publication,
     Distribution,
     Remote,
     Repository,
 )
 
+from pulp_gem.specs import analyse_gem
+
 
 log = getLogger(__name__)
 
@@ -42,6 +47,28 @@ def gemspec_path(self):
         """The path for this gem's gemspec for the content app."""
         return f"quick/Marshal.4.8/{self.name}-{self.version}.gemspec.rz"
 
+    @staticmethod
+    def init_from_artifact_and_relative_path(artifact, relative_path):
+        """"""
+        name, version, spec_data = analyse_gem(artifact.file)
+        content = GemContent(name=name, version=version)
+        relative_path = content.relative_path
+
+        with NamedTemporaryFile(mode="wb", dir=".", delete=False) as temp_file:
+            temp_file.write(spec_data)
+            temp_file.flush()
+
+        spec_artifact = Artifact.init_and_validate(temp_file.name)
+        try:
+            spec_artifact.save()
+        except utils.IntegrityError:
+            spec_artifact = Artifact.objects.get(spec_artifact.q())
+            spec_artifact.touch()
+        spec_relative_path = content.gemspec_path
+
+        artifacts = {relative_path: artifact, spec_relative_path: spec_artifact}
+        return content, artifacts
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("name", "version")
@@ -76,6 +103,20 @@ class GemRemote(Remote):
 
     TYPE = "gem"
 
+    def get_remote_artifact_content_type(self, relative_path=None):
+        """
+        Return a modified GemContent class that has a reference to this remote.
+
+        This will ensure that GemContent.init_from_artifact_and_relative_path can properly create
+        the Remote Artifact for the second Artifact it needs whether that be the gem file or the
+        gemspec.
+        """
+        if relative_path:
+            path = PurePath(relative_path)
+            if path.match("gems/*.gem"):
+                return GemContent
+        return None
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
 

pulp_gem/app/serializers.py

@@ -195,7 +195,14 @@ class GemDistributionSerializer(DistributionSerializer):
         queryset=Publication.objects.exclude(complete=False),
         allow_null=True,
     )
+    remote = DetailRelatedField(
+        required=False,
+        help_text=_("Remote that can be used to fetch content when using pull-through caching."),
+        view_name_pattern=r"remotes(-.*/.*)?-detail",
+        queryset=Remote.objects.all(),
+        allow_null=True,
+    )
 
     class Meta:
-        fields = DistributionSerializer.Meta.fields + ("publication",)
+        fields = DistributionSerializer.Meta.fields + ("publication", "remote")
         model = GemDistribution

pulp_gem/tests/functional/api/test_full_mirror.py

@@ -0,0 +1,77 @@
+import pytest
+import subprocess
+from aiohttp.client_exceptions import ClientResponseError
+
+
+def test_pull_through_metadata(
+    gem_remote_factory,
+    gem_distribution_factory,
+    gem_content_api_client,
+    artifacts_api_client,
+    http_get,
+):
+    """
+    Test that pull-through caching can retrieve metadata files upstream, but does not save them.
+    """
+    artifacts_before = artifacts_api_client.list().count
+    content_before = gem_content_api_client.list().count
+
+    # Choose a remote source that supports all the different gem repository metadata formats
+    remote = gem_remote_factory(url="https://rubygems.org")
+    distribution = gem_distribution_factory(remote=remote.pulp_href)
+
+    urls = [
+        "info/pulp_file_client",
+        "specs.4.8",
+        "latest_specs.4.8",
+        "prerelease_specs.4.8",
+        "quick/Marshal.4.8/pulp_file_client-1.14.0.gemspec.rz",
+    ]
+    for path in urls:
+        url = distribution.base_url + path
+        assert http_get(url)
+
+    artifacts_after = artifacts_api_client.list().count
+    content_after = gem_content_api_client.list().count
+
+    assert artifacts_before == artifacts_after
+    assert content_before == content_after
+
+    with pytest.raises(ClientResponseError) as e:
+        http_get(distribution.base_url + "NOT_A_VALID_LINK")
+
+    assert e.value.status == 404
+
+
+def test_pull_through_install(
+    gem_remote_factory, gem_distribution_factory, gem_content_api_client, delete_orphans_pre
+):
+    """
+    Test that gem clients can install from a distribution with pull-through caching.
+    """
+    out = subprocess.run(("which", "gem"))
+    if out.returncode != 0:
+        pytest.skip("gem not installed on test machine")
+    content_before = gem_content_api_client.list().count
+
+    remote = gem_remote_factory(url="https://rubygems.org")
+    distribution = gem_distribution_factory(remote=remote.pulp_href)
+
+    remote2 = gem_remote_factory()
+    distribution2 = gem_distribution_factory(remote=remote2.pulp_href)
+
+    for dis, gem in zip((distribution, distribution2), ("a", "beryl")):
+        cmd = ["gem", "i", "--remote", "--clear-sources", "-s", dis.base_url, gem, "-v", "0.1.0"]
+
+        out = subprocess.run(cmd, stdout=subprocess.PIPE)
+        assert f"Successfully installed {gem}-0.1.0" in out.stdout.decode("utf-8")
+
+        r = gem_content_api_client.list(name=gem, version="0.1.0")
+        assert r.count == 1
+        assert r.results[0].name == gem
+        assert r.results[0].version == "0.1.0"
+
+        subprocess.run(("gem", "uninstall", gem, "-v", "0.1.0"))
+
+    content_after = gem_content_api_client.list().count
+    assert content_before + 2 == content_after

requirements.txt

@@ -1,2 +1,2 @@
-pulpcore>=3.25.0,<3.40
+pulpcore>=3.28.0,<3.40
 rubymarshal==1.0.3