Update Pypi.dependencies after update to package manager api

app/models/package_manager/base.rb

@@ -235,7 +235,10 @@ def self.save_dependencies(mapped_project, sync_version: :all)
 
         deps = begin
           dependencies(name, db_version.number, mapped_project)
-        rescue StandardError
+        rescue StandardError => e
+          Rails.logger.error(
+            "Error while trying to get dependencies for #{db_platform}/#{name}@#{db_version.number}: #{e.message}"
+          )
           []
         end
 

app/models/package_manager/pypi.rb

@@ -11,6 +11,8 @@ class Pypi < Base
     ENTIRE_PACKAGE_CAN_BE_DEPRECATED = true
     SUPPORTS_SINGLE_VERSION_UPDATE = true
     PYPI_PRERELEASE = /(a|b|rc|dev)[0-9]+$/.freeze
+    # Adapted from https://peps.python.org/pep-0508/#names
+    PEP_508_NAME_REGEX = /^([A-Z0-9][A-Z0-9._-]*[A-Z0-9]|[A-Z0-9])/i
 
     def self.package_link(db_project, version = nil)
       # NB PEP 503: "All URLs which respond with an HTML5 page MUST end with a / and the repository SHOULD redirect the URLs without a / to add a / to the end."
@@ -123,17 +125,25 @@ def self.known_versions(name)
         &.index_by { |v| v[:number] } || {}
     end
 
-    def self.dependencies(name, version, _mapped_project)
+    # Simply parses out the name of a PEP 508 Dependency specification: https://peps.python.org/pep-0508/
+    # Leaves the rest as-is with any leading semicolons or spaces stripped
+    def self.parse_pep_508_dep_spec(dep)
+      name, requirement = dep.split(PEP_508_NAME_REGEX, 2).last(2).map(&:strip)
+      requirement = requirement.sub(/^[\s;]*/, "")
+      return name, requirement
+    end
+
+    def self.dependencies(name, version, _mapped_project = nil)
       api_response = get("https://pypi.org/pypi/#{name}/#{version}/json")
       deps = api_response.dig("info", "requires_dist")
-      source_info = api_response.dig("releases", version)
+      source_info = api_response.fetch("urls", [])
       Rails.logger.warn("Pypi sdist (no deps): #{name}") unless source_info.any? { |rel| rel["packagetype"] == "bdist_wheel" }
 
       deps.map do |dep|
-        name, version = dep.split
+        dep_name, requirements = parse_pep_508_dep_spec(dep)
         {
-          project_name: name,
-          requirements: version.nil? || version == ";" ? "*" : version.gsub(/\(|\)/, ""),
+          project_name: dep_name,
+          requirements: requirements.blank? ? "*" : requirements,
           kind: "runtime",
           optional: false,
           platform: self.name.demodulize,

spec/models/package_manager/pypi_spec.rb

@@ -119,4 +119,85 @@
       expect(described_class.deprecation_info('foo')).to eq({is_deprecated: true, message: "Development Status :: 7 - Inactive"})
     end
   end
+
+  describe ".dependencies" do
+    it "returns the dependencies of a particular version" do
+      VCR.use_cassette("pypi_dependencies_requests", record: :once) do
+        expect(
+          described_class.dependencies("requests", "2.28.2")
+        ).to match_array(
+               [
+                 ["charset-normalizer", "(<4,>=2)"],
+                 ["idna", "(<4,>=2.5)"],
+                 ["urllib3", "(<1.27,>=1.21.1)"],
+                 ["certifi", "(>=2017.4.17)"],
+                 ["PySocks", "(!=1.5.7,>=1.5.6) ; extra == 'socks'"],
+                 ["chardet", "(<6,>=3.0.2) ; extra == 'use_chardet_on_py3'"]
+               ].map do |name, requirements|
+                 {
+                   project_name: name,
+                   requirements: requirements,
+                   kind: "runtime",
+                   optional: false,
+                   platform: "Pypi"
+                 }
+               end
+             )
+      end
+    end
+
+    # Copied from the tests of https://peps.python.org/pep-0508/#complete-grammar
+    [
+      ["A", "A", ""],
+      ["A>=3", "A", ">=3"],
+      ["A.B-C_D", "A.B-C_D", ""],
+      ["aa", "aa", ""],
+      ["name", "name", ""],
+      ["name<=1", "name", "<=1"],
+      ["name>=3", "name", ">=3"],
+      ["name>=3,<2", "name", ">=3,<2"],
+      ["name@http://foo.com", "name", "@http://foo.com"],
+      [
+        "name [fred,bar] @ http://foo.com ; python_version=='2.7'",
+        "name",
+        "[fred,bar] @ http://foo.com ; python_version=='2.7'"
+      ],
+      [
+        "name[quux, strange];python_version<'2.7' and platform_version=='2'",
+        "name",
+        "[quux, strange];python_version<'2.7' and platform_version=='2'"
+      ],
+      [
+        "name; os_name=='a' or os_name=='b'",
+        "name",
+        "os_name=='a' or os_name=='b'"
+      ],
+      [
+        "name; os_name=='a' and os_name=='b' or os_name=='c'",
+        "name",
+        "os_name=='a' and os_name=='b' or os_name=='c'"
+      ],
+      [
+        "name; os_name=='a' and (os_name=='b' or os_name=='c')",
+        "name",
+        "os_name=='a' and (os_name=='b' or os_name=='c')"
+      ],
+      [
+        "name; os_name=='a' or os_name=='b' and os_name=='c'",
+        "name",
+        "os_name=='a' or os_name=='b' and os_name=='c'"
+      ],
+      [
+        "name; (os_name=='a' or os_name=='b') and os_name=='c'",
+        "name",
+        "(os_name=='a' or os_name=='b') and os_name=='c'"
+      ],
+    ].each do |test, expected_name, expected_requirement|
+      it "#{test} should be parsed correctly" do
+        expect(
+          PackageManager::Pypi.parse_pep_508_dep_spec(test)
+        ).to eq([expected_name, expected_requirement])
+      end
+    end
+  end
 end