Add support for HTML indexes

Author: charliermarsh

Cargo.lock

@@ -75,6 +75,7 @@ target-lexicon = { version = "0.12.12" }
 tempfile = { version = "3.8.1" }
 textwrap = { version = "0.15.2" }
 thiserror = { version = "1.0.50" }
+tl = { version = "0.7.7" }
 tokio = { version = "1.33.0", features = ["rt-multi-thread"] }
 tokio-util = { version = "0.7.10", features = ["compat"] }
 toml = { version = "0.8.6" }

Cargo.toml

@@ -8,7 +8,8 @@ use thiserror::Error;
 use pep440_rs::Version;
 use puffin_normalize::{InvalidNameError, PackageName};
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub enum SourceDistExtension {
     Zip,
     TarGz,

crates/distribution-filename/src/source_dist.rs

@@ -2701,3 +2701,42 @@ fn cache_errors_are_non_fatal() -> Result<()> {
         Ok(())
     }
 }
+
+/// Resolve a distribution from an HTML-only registry.
+#[test]
+fn compile_html() -> Result<()> {
+    let temp_dir = TempDir::new()?;
+    let cache_dir = TempDir::new()?;
+    let venv = create_venv_py312(&temp_dir, &cache_dir);
+
+    let requirements_in = temp_dir.child("requirements.in");
+    requirements_in.write_str("jinja2<=3.1.2")?;
+
+    insta::with_settings!({
+        filters => INSTA_FILTERS.to_vec()
+    }, {
+        assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
+            .arg("pip-compile")
+            .arg("requirements.in")
+            .arg("--cache-dir")
+            .arg(cache_dir.path())
+            .arg("--index-url")
+            .arg("https://download.pytorch.org/whl")
+            .env("VIRTUAL_ENV", venv.as_os_str())
+            .current_dir(&temp_dir), @r###"
+        success: true
+        exit_code: 0
+        ----- stdout -----
+        # This file was autogenerated by Puffin v0.0.1 via the following command:
+        #    puffin pip-compile requirements.in --cache-dir [CACHE_DIR]
+        jinja2==3.1.2
+        markupsafe==2.1.3
+            # via jinja2
+
+        ----- stderr -----
+        Resolved 2 packages in [TIME]
+        "###);
+    });
+
+    Ok(())
+}

crates/puffin-cli/tests/pip_compile.rs

@@ -4,7 +4,7 @@ version = "0.0.1"
 edition = "2021"
 
 [dependencies]
-distribution-filename = { path = "../distribution-filename" }
+distribution-filename = { path = "../distribution-filename", features = ["serde"] }
 distribution-types = { path = "../distribution-types" }
 install-wheel-rs = { path = "../install-wheel-rs" }
 pep440_rs = { path = "../pep440-rs" }
@@ -28,6 +28,7 @@ serde_json = { workspace = true }
 sha2 = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
+tl = { workspace = true }
 tokio = { workspace = true, features = ["fs"] }
 tokio-util = { workspace = true }
 tracing = { workspace = true }
@@ -37,4 +38,5 @@ url = { workspace = true }
 pep508_rs = { path = "../pep508-rs" }
 
 anyhow = { workspace = true }
+insta = { version = "1.34.0" }
 tokio = { workspace = true, features = ["fs", "macros"] }

crates/puffin-client/Cargo.toml

@@ -5,6 +5,7 @@ use async_zip::error::ZipError;
 use thiserror::Error;
 use url::Url;
 
+use crate::html;
 use distribution_filename::{WheelFilename, WheelFilenameError};
 use puffin_normalize::PackageName;
 
@@ -49,6 +50,9 @@ pub enum Error {
     #[error("Received some unexpected JSON from {url}")]
     BadJson { source: serde_json::Error, url: Url },
 
+    #[error("Received some unexpected HTML from {url}")]
+    BadHtml { source: html::Error, url: Url },
+
     #[error(transparent)]
     AsyncHttpRangeReader(#[from] AsyncHttpRangeReaderError),
 
@@ -82,10 +86,23 @@ pub enum Error {
     /// An [`io::Error`] with a filename attached
     #[error(transparent)]
     Persist(#[from] tempfile::PersistError),
+
+    #[error("Missing `Content-Type` header for {0}")]
+    MissingContentType(Url),
+
+    #[error("Invalid `Content-Type` header for {0}")]
+    InvalidContentTypeHeader(Url, #[source] http::header::ToStrError),
+
+    #[error("Unsupported `Content-Type` \"{1}\" for {0}")]
+    UnsupportedMediaType(Url, String),
 }
 
 impl Error {
-    pub fn from_json_err(err: serde_json::Error, url: Url) -> Self {
+    pub(crate) fn from_json_err(err: serde_json::Error, url: Url) -> Self {
         Self::BadJson { source: err, url }
     }
+
+    pub(crate) fn from_html_err(err: html::Error, url: Url) -> Self {
+        Self::BadHtml { source: err, url }
+    }
 }