Classify the transient request errors in one place
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -641,32 +641,44 @@ class ArtistFetcher:
|
|||||||
time.sleep(SLEEP_SECONDS)
|
time.sleep(SLEEP_SECONDS)
|
||||||
self.__sent += 1
|
self.__sent += 1
|
||||||
attempt: int = 1
|
attempt: int = 1
|
||||||
reason: str
|
reason: str | None
|
||||||
cause: BaseException
|
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(
|
with urllib.request.urlopen(
|
||||||
request, timeout=timeout) as response:
|
request, timeout=timeout) as response:
|
||||||
return response.read()
|
return response.read()
|
||||||
except urllib.error.HTTPError as error:
|
except (urllib.error.HTTPError, TimeoutError,
|
||||||
if error.code not in RETRY_STATUSES:
|
urllib.error.URLError) as error:
|
||||||
|
reason = self.__retry_reason(error)
|
||||||
|
if reason is None:
|
||||||
raise
|
raise
|
||||||
reason = str(error)
|
|
||||||
cause = error
|
|
||||||
except TimeoutError as error:
|
|
||||||
reason = str(error) or "timed out"
|
|
||||||
cause = error
|
|
||||||
except urllib.error.URLError as error:
|
|
||||||
if not isinstance(error.reason, TimeoutError):
|
|
||||||
raise
|
|
||||||
reason = str(error.reason) or "timed out"
|
|
||||||
cause = error
|
|
||||||
if attempt >= MAX_ATTEMPTS:
|
if attempt >= MAX_ATTEMPTS:
|
||||||
raise RetryExhausted(
|
raise RetryExhausted(
|
||||||
f"retries exhausted ({reason})") from cause
|
f"retries exhausted ({reason})") from error
|
||||||
time.sleep(RETRY_SECONDS * attempt)
|
time.sleep(RETRY_SECONDS * attempt)
|
||||||
attempt += 1
|
attempt += 1
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def __retry_reason(
|
||||||
|
error: urllib.error.URLError | TimeoutError) \
|
||||||
|
-> str | None:
|
||||||
|
"""Tell whether an error is transient, and why.
|
||||||
|
|
||||||
|
:param error: The HTTP or network error raised by the
|
||||||
|
request.
|
||||||
|
:return: The reason to report on the error, or None when
|
||||||
|
the error is not transient and must not be retried.
|
||||||
|
"""
|
||||||
|
if isinstance(error, urllib.error.HTTPError):
|
||||||
|
if error.code not in RETRY_STATUSES:
|
||||||
|
return None
|
||||||
|
return str(error)
|
||||||
|
if isinstance(error, TimeoutError):
|
||||||
|
return str(error) or "timed out"
|
||||||
|
if not isinstance(error.reason, TimeoutError):
|
||||||
|
return None
|
||||||
|
return str(error.reason) or "timed out"
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def __literals(texts: Sequence[str]) -> str:
|
def __literals(texts: Sequence[str]) -> str:
|
||||||
"""Build the SPARQL literals of texts at both languages.
|
"""Build the SPARQL literals of texts at both languages.
|
||||||
|
|||||||
Reference in New Issue
Block a user