Harden ZIP streaming to reject repeated entries and other malformed ZIP files (#15136)

## Summary

uv will now reject ZIP files that meet any of the following conditions:

- Multiple local header entries exist for the same file with different
contents.
- A local header entry exists for a file that isn't included in the
end-of-central directory record.
- An entry exists in the end-of-central directory record that does not
have a corresponding local header.
- The ZIP file contains contents after the first end-of-central
directory record.
- The CRC32 doesn't match between the local file header and the
end-of-central directory record.
- The compressed size doesn't match between the local file header and
the end-of-central directory record.
- The uncompressed size doesn't match between the local file header and
the end-of-central directory record.
- The reported central directory offset (in the end-of-central-directory
header) does not match the actual offset.
- The reported ZIP64 end of central directory locator offset does not
match the actual offset.

We also validate the above for files with data descriptors, which we
previously ignored.

Wheels from the most recent releases of the top 15,000 packages on PyPI
have been confirmed to pass these checks, and PyPI will also reject ZIPs
under many of the same conditions (at upload time) in the future.

In rare cases, this validation can be disabled by setting
`UV_INSECURE_NO_ZIP_VALIDATION=1`. Any validations should be reported to
the uv issue tracker and to the upstream package maintainer.

Author: charliermarsh

Cargo.lock

@@ -82,7 +82,7 @@ async-channel = { version = "2.3.1" }
 async-compression = { version = "0.4.12", features = ["bzip2", "gzip", "xz", "zstd"] }
 async-trait = { version = "0.1.82" }
 async_http_range_reader = { version = "0.9.1" }
-async_zip = { git = "https://github.com/astral-sh/rs-async-zip", rev = "c909fda63fcafe4af496a07bfda28a5aae97e58d", features = ["bzip2", "deflate", "lzma", "tokio", "xz", "zstd"] }
+async_zip = { git = "https://github.com/astral-sh/rs-async-zip", rev = "285e48742b74ab109887d62e1ae79e7c15fd4878", features = ["bzip2", "deflate", "lzma", "tokio", "xz", "zstd"] }
 axoupdater = { version = "0.9.0", default-features = false }
 backon = { version = "1.3.0" }
 base64 = { version = "0.22.1" }

Cargo.toml

@@ -22,7 +22,7 @@ uv-client = { workspace = true }
 uv-configuration = { workspace = true }
 uv-distribution-filename = { workspace = true }
 uv-distribution-types = { workspace = true }
-uv-extract = { workspace = true, optional = true }
+uv-extract = { workspace = true }
 uv-installer = { workspace = true }
 uv-macros = { workspace = true }
 uv-options-metadata = { workspace = true }
@@ -39,20 +39,23 @@ anstream = { workspace = true }
 anyhow = { workspace = true }
 clap = { workspace = true, features = ["derive", "wrap_help"] }
 fs-err = { workspace = true, features = ["tokio"] }
+futures = { workspace = true }
 itertools = { workspace = true }
 markdown = { version = "1.0.0" }
 owo-colors = { workspace = true }
 poloto = { version = "19.1.2", optional = true }
 pretty_assertions = { version = "1.4.1" }
-reqwest = { workspace = true }
+reqwest = { workspace = true, features = ["stream"] }
 resvg = { version = "0.29.0", optional = true }
 schemars = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml = { version = "0.9.34" }
 tagu = { version = "0.1.6", optional = true }
+tempfile = { workspace = true }
 textwrap = { workspace = true }
 tokio = { workspace = true }
+tokio-util = { workspace = true }
 tracing = { workspace = true }
 tracing-durations-export = { workspace = true, features = ["plot"] }
 tracing-subscriber = { workspace = true }

crates/uv-dev/Cargo.toml

@@ -14,6 +14,7 @@ use crate::generate_options_reference::Args as GenerateOptionsReferenceArgs;
 use crate::generate_sysconfig_mappings::Args as GenerateSysconfigMetadataArgs;
 #[cfg(feature = "render")]
 use crate::render_benchmarks::RenderBenchmarksArgs;
+use crate::validate_zip::ValidateZipArgs;
 use crate::wheel_metadata::WheelMetadataArgs;
 
 mod clear_compile;
@@ -25,6 +26,7 @@ mod generate_json_schema;
 mod generate_options_reference;
 mod generate_sysconfig_mappings;
 mod render_benchmarks;
+mod validate_zip;
 mod wheel_metadata;
 
 const ROOT_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../");
@@ -33,6 +35,8 @@ const ROOT_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../");
 enum Cli {
     /// Display the metadata for a `.whl` at a given URL.
     WheelMetadata(WheelMetadataArgs),
+    /// Validate that a `.whl` or `.zip` file at a given URL is a valid ZIP file.
+    ValidateZip(ValidateZipArgs),
     /// Compile all `.py` to `.pyc` files in the tree.
     Compile(CompileArgs),
     /// Remove all `.pyc` in the tree.
@@ -59,6 +63,7 @@ pub async fn run() -> Result<()> {
     let cli = Cli::parse();
     match cli {
         Cli::WheelMetadata(args) => wheel_metadata::wheel_metadata(args).await?,
+        Cli::ValidateZip(args) => validate_zip::validate_zip(args).await?,
         Cli::Compile(args) => compile::compile(args).await?,
         Cli::ClearCompile(args) => clear_compile::clear_compile(&args)?,
         Cli::GenerateAll(args) => generate_all::main(&args).await?,

crates/uv-dev/src/lib.rs

@@ -0,0 +1,43 @@
+use std::ops::Deref;
+
+use anyhow::{Result, bail};
+use clap::Parser;
+use futures::TryStreamExt;
+use tokio_util::compat::FuturesAsyncReadCompatExt;
+
+use uv_cache::{Cache, CacheArgs};
+use uv_client::RegistryClientBuilder;
+use uv_pep508::VerbatimUrl;
+use uv_pypi_types::ParsedUrl;
+
+#[derive(Parser)]
+pub(crate) struct ValidateZipArgs {
+    url: VerbatimUrl,
+    #[command(flatten)]
+    cache_args: CacheArgs,
+}
+
+pub(crate) async fn validate_zip(args: ValidateZipArgs) -> Result<()> {
+    let cache = Cache::try_from(args.cache_args)?.init()?;
+    let client = RegistryClientBuilder::new(cache).build();
+
+    let ParsedUrl::Archive(archive) = ParsedUrl::try_from(args.url.to_url())? else {
+        bail!("Only archive URLs are supported");
+    };
+
+    let response = client
+        .uncached_client(&archive.url)
+        .get(archive.url.deref().clone())
+        .send()
+        .await?;
+    let reader = response
+        .bytes_stream()
+        .map_err(std::io::Error::other)
+        .into_async_read();
+
+    let target = tempfile::TempDir::new()?;
+
+    uv_extract::stream::unzip(reader.compat(), target.path()).await?;
+
+    Ok(())
+}

crates/uv-dev/src/validate_zip.rs

@@ -19,6 +19,7 @@ workspace = true
 uv-configuration = { workspace = true }
 uv-distribution-filename = { workspace = true }
 uv-pypi-types = { workspace = true }
+uv-static = { workspace = true }
 
 astral-tokio-tar = { workspace = true }
 async-compression = { workspace = true, features = ["bzip2", "gzip", "zstd", "xz"] }

crates/uv-extract/Cargo.toml

@@ -14,12 +14,79 @@ pub enum Error {
     NonSingularArchive(Vec<OsString>),
     #[error("The top-level of the archive must only contain a list directory, but it's empty")]
     EmptyArchive,
+    #[error("ZIP local header filename at offset {offset} does not use UTF-8 encoding")]
+    LocalHeaderNotUtf8 { offset: u64 },
+    #[error("ZIP central directory entry filename at index {index} does not use UTF-8 encoding")]
+    CentralDirectoryEntryNotUtf8 { index: u64 },
     #[error("Bad CRC (got {computed:08x}, expected {expected:08x}) for file: {}", path.display())]
     BadCrc32 {
         path: PathBuf,
         computed: u32,
         expected: u32,
     },
+    #[error("Bad uncompressed size (got {computed:08x}, expected {expected:08x}) for file: {}", path.display())]
+    BadUncompressedSize {
+        path: PathBuf,
+        computed: u64,
+        expected: u64,
+    },
+    #[error("Bad compressed size (got {computed:08x}, expected {expected:08x}) for file: {}", path.display())]
+    BadCompressedSize {
+        path: PathBuf,
+        computed: u64,
+        expected: u64,
+    },
+    #[error("ZIP file contains multiple entries with different contents for: {}", path.display())]
+    DuplicateLocalFileHeader { path: PathBuf },
+    #[error("ZIP file contains a local file header without a corresponding central-directory record entry for: {} ({offset})", path.display())]
+    MissingCentralDirectoryEntry { path: PathBuf, offset: u64 },
+    #[error("ZIP file contains an end-of-central-directory record entry, but no local file header for: {} ({offset}", path.display())]
+    MissingLocalFileHeader { path: PathBuf, offset: u64 },
+    #[error("ZIP file uses conflicting paths for the local file header at {} (got {}, expected {})", offset, local_path.display(), central_directory_path.display())]
+    ConflictingPaths {
+        offset: u64,
+        local_path: PathBuf,
+        central_directory_path: PathBuf,
+    },
+    #[error("ZIP file uses conflicting checksums for the local file header and central-directory record (got {local_crc32}, expected {central_directory_crc32}) for: {} ({offset})", path.display())]
+    ConflictingChecksums {
+        path: PathBuf,
+        offset: u64,
+        local_crc32: u32,
+        central_directory_crc32: u32,
+    },
+    #[error("ZIP file uses conflicting compressed sizes for the local file header and central-directory record (got {local_compressed_size}, expected {central_directory_compressed_size}) for: {} ({offset})", path.display())]
+    ConflictingCompressedSizes {
+        path: PathBuf,
+        offset: u64,
+        local_compressed_size: u64,
+        central_directory_compressed_size: u64,
+    },
+    #[error("ZIP file uses conflicting uncompressed sizes for the local file header and central-directory record (got {local_uncompressed_size}, expected {central_directory_uncompressed_size}) for: {} ({offset})", path.display())]
+    ConflictingUncompressedSizes {
+        path: PathBuf,
+        offset: u64,
+        local_uncompressed_size: u64,
+        central_directory_uncompressed_size: u64,
+    },
+    #[error("ZIP file contains trailing contents after the end-of-central-directory record")]
+    TrailingContents,
+    #[error(
+        "ZIP file reports a number of entries in the central directory that conflicts with the actual number of entries (got {actual}, expected {expected})"
+    )]
+    ConflictingNumberOfEntries { actual: u64, expected: u64 },
+    #[error("Data descriptor is missing for file: {}", path.display())]
+    MissingDataDescriptor { path: PathBuf },
+    #[error("File contains an unexpected data descriptor: {}", path.display())]
+    UnexpectedDataDescriptor { path: PathBuf },
+    #[error(
+        "ZIP file end-of-central-directory record contains a comment that appears to be an embedded ZIP file"
+    )]
+    ZipInZip,
+    #[error("ZIP64 end-of-central-directory record contains unsupported extensible data")]
+    ExtensibleData,
+    #[error("ZIP file end-of-central-directory record contains multiple entries with the same path, but conflicting modes: {}", path.display())]
+    DuplicateExecutableFileHeader { path: PathBuf },
 }
 
 impl Error {