Introduce resumable downloads with --resume-retries (#12991)

Add the option --resume-retries to allow pip to retry a download X
number of times.

When a download times out or fails midway through and the download
size is known, a HTTP range request will be issued to resume the
download. If the server supports a partial download (i.e. returns 206
Partial Content), the download will be resumed. If the server responds
with 200 (i.e., the server does NOT support range requests or the file
has changed since the last request), we fall back to restarting the
download. This repeats as necessary until the download is complete or
no more retries remain.

The number of resumes currently defaults to zero, but it is expected to
be increased sometime after the 25.1 release once we've gotten feedback.

Signed-off-by: gmargaritis <[email protected]>
Co-authored-by: Yichi Yang <[email protected]>
Co-authored-by: Richard Si <[email protected]>

Author: 3 people

news/12991.feature.rst

@@ -0,0 +1,3 @@
+Add support to enable resuming incomplete downloads.
+
+Control the number of retry attempts using the ``--resume-retries`` flag.

src/pip/_internal/cli/cmdoptions.py

@@ -281,8 +281,17 @@ class PipOption(Option):
     dest="retries",
     type="int",
     default=5,
-    help="Maximum number of retries each connection should attempt "
-    "(default %default times).",
+    help="Maximum attempts to establish a new HTTP connection. (default: %default)",
+)
+
+resume_retries: Callable[..., Option] = partial(
+    Option,
+    "--resume-retries",
+    dest="resume_retries",
+    type="int",
+    default=0,
+    help="Maximum attempts to resume or restart an incomplete download. "
+    "(default: %default)",
 )
 
 timeout: Callable[..., Option] = partial(
@@ -1077,7 +1086,6 @@ def check_list_path_option(options: Values) -> None:
     help=("Enable deprecated functionality, that will be removed in the future."),
 )
 
-
 ##########
 # groups #
 ##########
@@ -1110,6 +1118,7 @@ def check_list_path_option(options: Values) -> None:
         no_python_version_warning,
         use_new_feature,
         use_deprecated_feature,
+        resume_retries,
     ],
 }
 

src/pip/_internal/cli/progress_bars.py

@@ -29,6 +29,7 @@ def _rich_download_progress_bar(
     *,
     bar_type: str,
     size: Optional[int],
+    initial_progress: Optional[int] = None,
 ) -> Generator[bytes, None, None]:
     assert bar_type == "on", "This should only be used in the default mode."
 
@@ -54,6 +55,8 @@ def _rich_download_progress_bar(
 
     progress = Progress(*columns, refresh_per_second=5)
     task_id = progress.add_task(" " * (get_indentation() + 2), total=total)
+    if initial_progress is not None:
+        progress.update(task_id, advance=initial_progress)
     with progress:
         for chunk in iterable:
             yield chunk
@@ -86,12 +89,13 @@ def _raw_progress_bar(
     iterable: Iterable[bytes],
     *,
     size: Optional[int],
+    initial_progress: Optional[int] = None,
 ) -> Generator[bytes, None, None]:
     def write_progress(current: int, total: int) -> None:
         sys.stdout.write(f"Progress {current} of {total}\n")
         sys.stdout.flush()
 
-    current = 0
+    current = initial_progress or 0
     total = size or 0
     rate_limiter = RateLimiter(0.25)
 
@@ -105,18 +109,25 @@ def write_progress(current: int, total: int) -> None:
 
 
 def get_download_progress_renderer(
-    *, bar_type: str, size: Optional[int] = None
+    *, bar_type: str, size: Optional[int] = None, initial_progress: Optional[int] = None
 ) -> ProgressRenderer[bytes]:
     """Get an object that can be used to render the download progress.
 
     Returns a callable, that takes an iterable to "wrap".
     """
     if bar_type == "on":
         return functools.partial(
-            _rich_download_progress_bar, bar_type=bar_type, size=size
+            _rich_download_progress_bar,
+            bar_type=bar_type,
+            size=size,
+            initial_progress=initial_progress,
         )
     elif bar_type == "raw":
-        return functools.partial(_raw_progress_bar, size=size)
+        return functools.partial(
+            _raw_progress_bar,
+            size=size,
+            initial_progress=initial_progress,
+        )
     else:
         return iter  # no-op, when passed an iterator
 

src/pip/_internal/cli/req_command.py

@@ -144,6 +144,7 @@ def make_requirement_preparer(
             lazy_wheel=lazy_wheel,
             verbosity=verbosity,
             legacy_resolver=legacy_resolver,
+            resume_retries=options.resume_retries,
         )
 
     @classmethod

src/pip/_internal/exceptions.py

@@ -27,6 +27,7 @@
     from pip._vendor.requests.models import Request, Response
 
     from pip._internal.metadata import BaseDistribution
+    from pip._internal.models.link import Link
     from pip._internal.req.req_install import InstallRequirement
 
 logger = logging.getLogger(__name__)
@@ -809,6 +810,38 @@ def __init__(
         )
 
 
+class IncompleteDownloadError(DiagnosticPipError):
+    """Raised when the downloader receives fewer bytes than advertised
+    in the Content-Length header."""
+
+    reference = "incomplete-download"
+
+    def __init__(
+        self, link: "Link", received: int, expected: int, *, retries: int
+    ) -> None:
+        # Dodge circular import.
+        from pip._internal.utils.misc import format_size
+
+        download_status = f"{format_size(received)}/{format_size(expected)}"
+        if retries:
+            retry_status = f"after {retries} attempts "
+            hint = "Use --resume-retries to configure resume attempt limit."
+        else:
+            retry_status = ""
+            hint = "Consider using --resume-retries to enable download resumption."
+        message = Text(
+            f"Download failed {retry_status}because not enough bytes "
+            f"were received ({download_status})"
+        )
+
+        super().__init__(
+            message=message,
+            context=f"URL: {link.redacted_url}",
+            hint_stmt=hint,
+            note_stmt="This is an issue with network connectivity, not pip.",
+        )
+
+
 class ResolutionTooDeepError(DiagnosticPipError):
     """Raised when the dependency resolver exceeds the maximum recursion depth."""
 

src/pip/_internal/models/link.py

@@ -380,9 +380,9 @@ def __str__(self) -> str:
         else:
             rp = ""
         if self.comes_from:
-            return f"{redact_auth_from_url(self._url)} (from {self.comes_from}){rp}"
+            return f"{self.redacted_url} (from {self.comes_from}){rp}"
         else:
-            return redact_auth_from_url(str(self._url))
+            return self.redacted_url
 
     def __repr__(self) -> str:
         return f"<Link {self}>"
@@ -404,6 +404,10 @@ def __lt__(self, other: Any) -> bool:
     def url(self) -> str:
         return self._url
 
+    @property
+    def redacted_url(self) -> str:
+        return redact_auth_from_url(self.url)
+
     @property
     def filename(self) -> str:
         path = self.path.rstrip("/")