Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

lfs: add tests for rate limit retries #341

Merged
merged 2 commits into from
Mar 16, 2024
Merged

lfs: add tests for rate limit retries #341

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e04d43
lfs: add tests for rate limit retries
sisp Mar 15, 2024
ee61871
lfs: refactor rate limit retries tests with factored out LFS mocks
sisp Mar 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ Source = "https://github.com/iterative/scmrepo"

[project.optional-dependencies]
tests = [
"aioresponses==0.7.6",
"pytest==8.1.1",
"pytest-sugar==1.0.0",
"pytest-cov==4.1.0",
Expand Down
2 changes: 1 addition & 1 deletion src/scmrepo/git/lfs/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def loop(self):
def from_git_url(cls, git_url: str) -> "LFSClient":
if git_url.startswith(("ssh://", "git@")):
return _SSHLFSClient.from_git_url(git_url)
if git_url.startswith("https://"):
if git_url.startswith(("http://", "https://")):
return _HTTPLFSClient.from_git_url(git_url)
raise NotImplementedError(f"Unsupported Git URL: {git_url}")

Expand Down
209 changes: 207 additions & 2 deletions tests/test_lfs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,27 @@
# pylint: disable=redefined-outer-name
import io
from collections import defaultdict
from collections.abc import Sequence
from http import HTTPStatus
from time import time
from typing import Callable

import pytest
from aiohttp import ClientResponseError
from aioresponses import CallbackResult, aioresponses
from pytest_mock import MockerFixture
from pytest_test_utils import TempDirFactory, TmpDir
from yarl import URL

from scmrepo.git import Git
from scmrepo.git.lfs import LFSStorage, Pointer, smudge
from scmrepo.git.lfs import LFSClient, LFSStorage, Pointer, smudge

FOO_OID = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"
FOO_SIZE = 3
FOO_POINTER = (
f"version https://git-lfs.github.com/spec/v1\noid sha256:{FOO_OID}\nsize 3\n"
f"version https://git-lfs.github.com/spec/v1\n"
f"oid sha256:{FOO_OID}\n"
f"size {FOO_SIZE}\n"
).encode()


Expand Down Expand Up @@ -74,3 +85,197 @@ def test_lfs(tmp_dir: TmpDir, scm: Git, lfs_objects: TmpDir):
assert fobj.read() == FOO_POINTER
with fs.open("foo.lfs", "rb", raw=False) as fobj:
assert fobj.read() == b"foo"


class CallbackResultRecorder:
def __init__(self) -> None:
self._results: dict[str, list[CallbackResult]] = defaultdict(list)

def record(self, result: CallbackResult) -> Callable[..., CallbackResult]:
def _callback(url: URL, **_) -> CallbackResult:
self._results[str(url)].append(result)
return result

return _callback

def __getitem__(self, url: str) -> Sequence[CallbackResult]:
return self._results[url]


@pytest.mark.parametrize(
"rate_limit_header",
[
lambda: {"Retry-After": "1"},
lambda: {"RateLimit-Reset": f"{int(time()) + 1}"},
lambda: {"X-RateLimit-Reset": f"{int(time()) + 1}"},
],
)
def test_rate_limit_retry(
storage: LFSStorage, rate_limit_header: Callable[[], dict[str, str]]
):
client = LFSClient.from_git_url("http://git.example.com/namespace/project.git")
lfs_batch_url = f"{client.url}/objects/batch"
lfs_object_url = f"http://git-lfs.example.com/{FOO_OID}"
recorder = CallbackResultRecorder()

with aioresponses() as m:
m.post(
lfs_batch_url,
callback=recorder.record(
CallbackResult(
status=HTTPStatus.TOO_MANY_REQUESTS,
headers=rate_limit_header(),
reason="Too many requests",
)
),
)
m.post(
lfs_batch_url,
callback=recorder.record(
CallbackResult(
status=HTTPStatus.OK,
headers={"Content-Type": "application/vnd.git-lfs+json"},
payload={
"transfer": "basic",
"objects": [
{
"oid": FOO_OID,
"size": FOO_SIZE,
"authenticated": True,
"actions": {
"download": {
"href": lfs_object_url,
}
},
}
],
"hash_algo": "sha256",
},
)
),
)
m.get(
lfs_object_url,
callback=recorder.record(
CallbackResult(
status=HTTPStatus.TOO_MANY_REQUESTS,
headers=rate_limit_header(),
reason="Too many requests",
)
),
)
m.get(
lfs_object_url,
callback=recorder.record(
CallbackResult(
status=HTTPStatus.OK,
body="lfs data",
)
),
)

client.download(storage, [Pointer(oid=FOO_OID, size=FOO_SIZE)])

results = recorder[lfs_batch_url]
assert [r.status for r in results] == [429, 200]

results = recorder[lfs_object_url]
assert [r.status for r in results] == [429, 200]


@pytest.mark.parametrize(
"rate_limit_header",
[
lambda: {"Retry-After": "1"},
lambda: {"RateLimit-Reset": f"{int(time()) + 1}"},
lambda: {"X-RateLimit-Reset": f"{int(time()) + 1}"},
],
)
def test_rate_limit_max_retries_batch(
storage: LFSStorage, rate_limit_header: Callable[[], dict[str, str]]
):
client = LFSClient.from_git_url("http://git.example.com/namespace/project.git")
recorder = CallbackResultRecorder()

with aioresponses() as m:
m.post(
f"{client.url}/objects/batch",
callback=recorder.record(
CallbackResult(
status=HTTPStatus.TOO_MANY_REQUESTS,
headers=rate_limit_header(),
reason="Too many requests",
)
),
repeat=True,
)

with pytest.raises(ClientResponseError, match="Too many requests"):
client.download(storage, [Pointer(oid=FOO_OID, size=FOO_SIZE)])

results = recorder[f"{client.url}/objects/batch"]
assert [r.status for r in results] == [429] * 5


@pytest.mark.parametrize(
"rate_limit_header",
[
lambda: {"Retry-After": "1"},
lambda: {"RateLimit-Reset": f"{int(time()) + 1}"},
lambda: {"X-RateLimit-Reset": f"{int(time()) + 1}"},
],
)
def test_rate_limit_max_retries_objects(
storage: LFSStorage, rate_limit_header: Callable[[], dict[str, str]]
):
client = LFSClient.from_git_url("http://git.example.com/namespace/project.git")
lfs_batch_url = f"{client.url}/objects/batch"
lfs_object_url = f"http://git-lfs.example.com/{FOO_OID}"
recorder = CallbackResultRecorder()

with aioresponses() as m:
m.post(
lfs_batch_url,
callback=recorder.record(
CallbackResult(
status=HTTPStatus.OK,
headers={"Content-Type": "application/vnd.git-lfs+json"},
payload={
"transfer": "basic",
"objects": [
{
"oid": FOO_OID,
"size": FOO_SIZE,
"authenticated": True,
"actions": {
"download": {
"href": lfs_object_url,
}
},
}
],
"hash_algo": "sha256",
},
)
),
)
m.get(
lfs_object_url,
callback=recorder.record(
CallbackResult(
status=HTTPStatus.TOO_MANY_REQUESTS,
headers=rate_limit_header(),
reason="Too many requests",
),
),
repeat=True,
)

with pytest.raises(ClientResponseError, match="Too many requests"):
client.download(storage, [Pointer(oid=FOO_OID, size=FOO_SIZE)])

results = recorder[lfs_batch_url]
assert [r.status for r in results] == [200]

results = recorder[lfs_object_url]
assert [r.status for r in results] == [429] * 5
Loading