diff --git a/ext/REPLExt/completions.jl b/ext/REPLExt/completions.jl index c96b762ebe..fdd09e55dc 100644 --- a/ext/REPLExt/completions.jl +++ b/ext/REPLExt/completions.jl @@ -78,32 +78,30 @@ function complete_remote_package!(comps, partial; hint::Bool) name = regpkg.name name in cmp && continue if startswith(regpkg.name, partial) - pkg = Registry.registry_info(regpkg) + pkg = Registry.registry_info(reg, regpkg) Registry.isdeprecated(pkg) && continue - compat_info = Registry.compat_info(pkg) - # Filter versions - for (v, uncompressed_compat) in compat_info + # Check if any non-yanked version is compatible with current Julia + found_compatible_version = false + for v in keys(pkg.version_info) Registry.isyanked(pkg, v) && continue # TODO: Filter based on offline mode - is_julia_compat = nothing - for (pkg_uuid, vspec) in uncompressed_compat - if pkg_uuid == JULIA_UUID - is_julia_compat = VERSION in vspec - is_julia_compat && continue - end - end - # Found a compatible version or compat on julia at all => compatible - if is_julia_compat === nothing || is_julia_compat - push!(cmp, name) - # In hint mode the result is only used if there is a single matching entry - # so we can return no matches in case of more than one match - if hint && found_match - return true # true means returned early - end - found_match = true + # Query compressed compat for this version (optimized: only fetch Julia compat) + julia_vspec = Pkg.Registry.query_compat_for_version(pkg, v, JULIA_UUID) + # Found a compatible version or no julia compat at all => compatible + if julia_vspec === nothing || VERSION in julia_vspec + found_compatible_version = true break end end + if found_compatible_version + push!(cmp, name) + # In hint mode the result is only used if there is a single matching entry + # so we can return no matches in case of more than one match + if hint && found_match + return true # true means returned early + end + found_match = true + end end end end diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 9e49cb8c25..805db06099 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -105,7 +105,7 @@ function get_max_version_register(pkg::PackageSpec, regs) if get(reg, pkg.uuid, nothing) !== nothing reg_pkg = get(reg, pkg.uuid, nothing) reg_pkg === nothing && continue - pkg_info = Registry.registry_info(reg_pkg) + pkg_info = Registry.registry_info(reg, reg_pkg) for (version, info) in pkg_info.version_info info.yanked && continue if pkg.version isa VersionNumber diff --git a/src/Operations.jl b/src/Operations.jl index 4d999a684f..519b017012 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -27,7 +27,7 @@ function is_pkgversion_yanked(uuid::UUID, version::VersionNumber, registries::Ve for reg in registries reg_pkg = get(reg, uuid, nothing) if reg_pkg !== nothing - info = Registry.registry_info(reg_pkg) + info = Registry.registry_info(reg, reg_pkg) if haskey(info.version_info, version) && Registry.isyanked(info, version) return true end @@ -55,7 +55,7 @@ function get_pkg_deprecation_info(pkg::Union{PackageSpec, PackageEntry}, registr for reg in registries reg_pkg = get(reg, pkg.uuid, nothing) if reg_pkg !== nothing - info = Registry.registry_info(reg_pkg) + info = Registry.registry_info(reg, reg_pkg) if Registry.isdeprecated(info) return info.deprecated end @@ -316,7 +316,7 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju for reg in registries reg_pkg = get(reg, pkg.uuid, nothing) reg_pkg === nothing && continue - pkg_info = Registry.registry_info(reg_pkg) + pkg_info = Registry.registry_info(reg, reg_pkg) version_info = get(pkg_info.version_info, pkg.version, nothing) version_info === nothing && continue push!(pkg_reg_uuids, reg.uuid) @@ -439,7 +439,7 @@ function load_tree_hash!( for reg in registries reg_pkg = get(reg, pkg.uuid, nothing) reg_pkg === nothing && continue - pkg_info = Registry.registry_info(reg_pkg) + pkg_info = Registry.registry_info(reg, reg_pkg) version_info = get(pkg_info.version_info, pkg.version, nothing) version_info === nothing && continue hash′ = version_info.git_tree_sha1 @@ -763,7 +763,8 @@ function resolve_versions!( # happened on a different julia version / commit and the stdlib version in the manifest is not the current stdlib version unbind_stdlibs = julia_version === VERSION reqs = Resolve.Requires(pkg.uuid => is_stdlib(pkg.uuid) && unbind_stdlibs ? VersionSpec("*") : VersionSpec(pkg.version) for pkg in pkgs) - graph, compat_map = deps_graph(env, registries, names, reqs, fixed, julia_version, installed_only) + deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed, pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed = deps_graph(env, registries, names, reqs, fixed, julia_version, installed_only) + graph = Resolve.Graph(deps_map_compressed, compat_map_compressed, weak_deps_map_compressed, weak_compat_map_compressed, pkg_versions_map, pkg_versions_per_registry, uuid_to_name, reqs, fixed, false, julia_version) Resolve.simplify_graph!(graph) vers = Resolve.resolve(graph) @@ -773,14 +774,14 @@ function resolve_versions!( old_v = get(jll_fix, uuid, nothing) # We only fixup a JLL if the old major/minor/patch matches the new major/minor/patch if old_v !== nothing && Base.thispatch(old_v) == Base.thispatch(vers_fix[uuid]) - new_v = vers_fix[uuid] - if old_v != new_v && haskey(compat_map[uuid], old_v) - compat_map[uuid][old_v] = compat_map[uuid][new_v] - # Note that we don't delete!(compat_map[uuid], old_v) because we want to keep the compat info around - # in case there's JLL version confusion between the sysimage pkgorigins version and manifest - # but that issue hasn't been fully specified, so keep it to be cautious - end vers_fix[uuid] = old_v + # Add old_v to pkg_versions_map so it's considered available + # even if it was yanked (needed for sysimage compatibility) + versions_for_pkg = get!(pkg_versions_map, uuid, VersionNumber[]) + if !(old_v in versions_for_pkg) + push!(versions_for_pkg, old_v) + sort!(versions_for_pkg) + end end end vers = vers_fix @@ -809,11 +810,15 @@ function resolve_versions!( deps_fixed else d = Dict{String, UUID}() - if !haskey(compat_map[pkg.uuid], pkg.version) - available_versions = sort!(collect(keys(compat_map[pkg.uuid]))) + available_versions = get(Vector{VersionNumber}, pkg_versions_map, pkg.uuid) + if !(pkg.version in available_versions) pkgerror("version $(pkg.version) of package $(pkg.name) is not available. Available versions: $(join(available_versions, ", "))") end - for (uuid, _) in compat_map[pkg.uuid][pkg.version] + deps_for_version = Registry.query_deps_for_version( + deps_map_compressed, weak_deps_map_compressed, + pkg.uuid, pkg.version + ) + for uuid in deps_for_version d[names[uuid]] = uuid end d @@ -853,12 +858,27 @@ function deps_graph( stdlibs_for_julia_version = Types.get_last_stdlibs(julia_version) seen = Set{UUID}() - # pkg -> version -> (dependency => compat): - all_compat = Dict{UUID, Dict{VersionNumber, Dict{UUID, VersionSpec}}}() - weak_compat = Dict{UUID, Dict{VersionNumber, Set{UUID}}}() + # pkg -> vector of (registry data) for handling multiple registries correctly + # Each element in the vector represents data from one registry + all_deps_compressed = Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}}() + all_compat_compressed = Dict{UUID, Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}}() + weak_deps_compressed = Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}}() + weak_compat_compressed = Dict{UUID, Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}}() + + # pkg -> list of valid versions: + pkg_versions = Dict{UUID, Vector{VersionNumber}}() + + # pkg -> vector of (versions from each registry) - parallel to the compressed data vectors + # This tracks which versions came from which registry to avoid cross-registry compat pollution + pkg_versions_per_registry = Dict{UUID, Vector{Set{VersionNumber}}}() for (fp, fx) in fixed - all_compat[fp] = Dict(fx.version => Dict{UUID, VersionSpec}()) + all_deps_compressed[fp] = [Dict{VersionRange, Set{UUID}}()] + all_compat_compressed[fp] = [Dict{VersionRange, Dict{UUID, VersionSpec}}()] + weak_deps_compressed[fp] = [Dict{VersionRange, Set{UUID}}()] + weak_compat_compressed[fp] = [Dict{VersionRange, Dict{UUID, VersionSpec}}()] + pkg_versions[fp] = [fx.version] + pkg_versions_per_registry[fp] = [Set([fx.version])] end while true @@ -867,8 +887,6 @@ function deps_graph( for uuid in unseen push!(seen, uuid) uuid in keys(fixed) && continue - all_compat_u = get_or_make!(all_compat, uuid) - weak_compat_u = get_or_make!(weak_compat, uuid) uuid_is_stdlib = haskey(stdlibs_for_julia_version, uuid) # If we're requesting resolution of a package that is an @@ -880,74 +898,107 @@ function deps_graph( stdlib_info = stdlibs_for_julia_version[uuid] v = something(stdlib_info.version, VERSION) - all_compat_u_vr = get_or_make!(all_compat_u, v) + # For stdlibs, create a single registry entry + stdlib_deps = Dict{VersionRange, Set{UUID}}() + stdlib_compat = Dict{VersionRange, Dict{UUID, VersionSpec}}() + stdlib_weak_deps = Dict{VersionRange, Set{UUID}}() + stdlib_weak_compat = Dict{VersionRange, Dict{UUID, VersionSpec}}() + + vrange = VersionRange(v, v) + deps_set = Set{UUID}() for other_uuid in stdlib_info.deps push!(uuids, other_uuid) - all_compat_u_vr[other_uuid] = VersionSpec() + push!(deps_set, other_uuid) end + stdlib_deps[vrange] = deps_set + stdlib_compat[vrange] = Dict{UUID, VersionSpec}() if !isempty(stdlib_info.weakdeps) - weak_all_compat_u_vr = get_or_make!(weak_compat_u, v) + weak_deps_set = Set{UUID}() for other_uuid in stdlib_info.weakdeps push!(uuids, other_uuid) - all_compat_u_vr[other_uuid] = VersionSpec() - push!(weak_all_compat_u_vr, other_uuid) + push!(weak_deps_set, other_uuid) end + stdlib_weak_deps[vrange] = weak_deps_set + stdlib_weak_compat[vrange] = Dict{UUID, VersionSpec}() end + + all_deps_compressed[uuid] = [stdlib_deps] + all_compat_compressed[uuid] = [stdlib_compat] + weak_deps_compressed[uuid] = [stdlib_weak_deps] + weak_compat_compressed[uuid] = [stdlib_weak_compat] + pkg_versions[uuid] = [v] + pkg_versions_per_registry[uuid] = [Set([v])] else + # Accumulate valid versions from all registries + valid_versions = VersionNumber[] + # Store per-registry data separately - don't merge! + pkg_deps_list = Vector{Dict{VersionRange, Set{UUID}}}() + pkg_compat_list = Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}() + pkg_weak_deps_list = Vector{Dict{VersionRange, Set{UUID}}}() + pkg_weak_compat_list = Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}() + pkg_versions_per_reg = Vector{Set{VersionNumber}}() + for reg in registries pkg = get(reg, uuid, nothing) pkg === nothing && continue - info = Registry.registry_info(pkg) - - function add_compat!(d, cinfo) - for (v, compat_info) in cinfo - # Filter yanked and if we are in offline mode also downloaded packages - # TODO, pull this into a function - Registry.isyanked(info, v) && continue - if installed_only - pkg_spec = PackageSpec(name = pkg.name, uuid = pkg.uuid, version = v, tree_hash = Registry.treehash(info, v)) - is_package_downloaded(env.manifest_file, pkg_spec) || continue - end + info = Registry.registry_info(reg, pkg) + + # Build filtered version list for this registry + reg_valid_versions = Set{VersionNumber}() + for v in keys(info.version_info) + # Filter yanked and if we are in offline mode also downloaded packages + Registry.isyanked(info, v) && continue + if installed_only + pkg_spec = PackageSpec(name = pkg.name, uuid = pkg.uuid, version = v, tree_hash = Registry.treehash(info, v)) + is_package_downloaded(env.manifest_file, pkg_spec) || continue + end - # Skip package version that are not the same as external packages in sysimage - if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && julia_version == VERSION - pkgid = Base.PkgId(uuid, pkg.name) - if Base.in_sysimage(pkgid) - pkgorigin = get(Base.pkgorigins, pkgid, nothing) - if pkgorigin !== nothing && pkgorigin.version !== nothing - if v != pkgorigin.version - continue - end - end - end - end - dv = get_or_make!(d, v) - # Filter out incompatible stdlib compat entries from registry dependencies - for (dep_uuid, dep_compat) in compat_info - if is_stdlib(dep_uuid) && !(dep_uuid in Types.UPGRADABLE_STDLIBS_UUIDS) - stdlib_ver = stdlib_version(dep_uuid, julia_version) - if stdlib_ver !== nothing && !isempty(dep_compat) && !(stdlib_ver in dep_compat) - @debug "Ignoring incompatible stdlib compat entry" dep = get(uuid_to_name, dep_uuid, string(dep_uuid)) stdlib_ver dep_compat registry = reg.name package = pkg.name version = v + # Skip package version that are not the same as external packages in sysimage + if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && julia_version == VERSION + pkgid = Base.PkgId(uuid, pkg.name) + if Base.in_sysimage(pkgid) + pkgorigin = get(Base.pkgorigins, pkgid, nothing) + if pkgorigin !== nothing && pkgorigin.version !== nothing + if v != pkgorigin.version continue end end - dv[dep_uuid] = dep_compat - push!(uuids, dep_uuid) end end - return + + push!(reg_valid_versions, v) + push!(valid_versions, v) end - add_compat!(all_compat_u, Registry.compat_info(info)) - weak_compat_info = Registry.weak_compat_info(info) - if weak_compat_info !== nothing - add_compat!(all_compat_u, weak_compat_info) - # Version to Set - for (v, compat_info) in weak_compat_info - weak_compat_u[v] = keys(compat_info) + + # Only add this registry's data if it has valid versions + if !isempty(reg_valid_versions) + # Store the full compressed data along with which versions are valid + # The query function will check version membership to avoid cross-registry pollution + push!(pkg_deps_list, info.deps) + push!(pkg_compat_list, info.compat) + push!(pkg_weak_deps_list, info.weak_deps) + push!(pkg_weak_compat_list, info.weak_compat) + push!(pkg_versions_per_reg, reg_valid_versions) + end + + # Collect all dependency UUIDs for discovery + for deps_dict in (info.deps, info.weak_deps) + for (vrange, deps_set) in deps_dict + union!(uuids, deps_set) end end end + + # After processing all registries, sort and store the accumulated versions + pkg_versions[uuid] = sort!(unique!(valid_versions)) + + # Store the per-registry data + all_deps_compressed[uuid] = pkg_deps_list + all_compat_compressed[uuid] = pkg_compat_list + weak_deps_compressed[uuid] = pkg_weak_deps_list + weak_compat_compressed[uuid] = pkg_weak_compat_list + pkg_versions_per_registry[uuid] = pkg_versions_per_reg end end end @@ -991,8 +1042,7 @@ function deps_graph( fixed = fixed_filtered end - return Resolve.Graph(all_compat, weak_compat, uuid_to_name, reqs, fixed, false, julia_version), - all_compat + return all_deps_compressed, all_compat_compressed, weak_deps_compressed, weak_compat_compressed, pkg_versions, pkg_versions_per_registry, uuid_to_name, reqs, fixed end ######################## @@ -1375,7 +1425,7 @@ function find_urls(registries::Vector{Registry.RegistryInstance}, uuid::UUID) for reg in registries reg_pkg = get(reg, uuid, nothing) reg_pkg === nothing && continue - info = Registry.registry_info(reg_pkg) + info = Registry.registry_info(reg, reg_pkg) repo = info.repo repo === nothing && continue push!(urls, repo) @@ -3025,14 +3075,14 @@ function status_compat_info(pkg::PackageSpec, env::EnvCache, regs::Vector{Regist for reg in regs reg_pkg = get(reg, pkg.uuid, nothing) reg_pkg === nothing && continue - info = Registry.registry_info(reg_pkg) - reg_compat_info = Registry.compat_info(info) - versions = keys(reg_compat_info) + info = Registry.registry_info(reg, reg_pkg) + # Get versions directly from version_info + versions = keys(info.version_info) versions = filter(v -> !Registry.isyanked(info, v), versions) max_version_reg = maximum(versions; init = v"0") max_version = max(max_version, max_version_reg) compat_spec = get_compat_workspace(env, pkg.name) - versions_in_compat = filter(in(compat_spec), keys(reg_compat_info)) + versions_in_compat = filter(in(compat_spec), versions) max_version_in_compat = max(max_version_in_compat, maximum(versions_in_compat; init = v"0")) end max_version == v"0" && return nothing @@ -3065,11 +3115,9 @@ function status_compat_info(pkg::PackageSpec, env::EnvCache, regs::Vector{Regist for reg in regs reg_pkg = get(reg, uuid, nothing) reg_pkg === nothing && continue - info = Registry.registry_info(reg_pkg) - reg_compat_info = Registry.compat_info(info) - compat_info_v = get(reg_compat_info, dep_info.version, nothing) - compat_info_v === nothing && continue - compat_info_v_uuid = get(compat_info_v, pkg.uuid, nothing) + info = Registry.registry_info(reg, reg_pkg) + # Query compressed deps and compat for the specific dependency version (optimized: only fetch this pkg's compat) + compat_info_v_uuid = Registry.query_compat_for_version(info, dep_info.version, pkg.uuid) compat_info_v_uuid === nothing && continue if !(max_version in compat_info_v_uuid) push!(packages_holding_back, dep_pkg.name) @@ -3082,15 +3130,11 @@ function status_compat_info(pkg::PackageSpec, env::EnvCache, regs::Vector{Regist for reg in regs reg_pkg = get(reg, pkg.uuid, nothing) reg_pkg === nothing && continue - info = Registry.registry_info(reg_pkg) - reg_compat_info = Registry.compat_info(info) - compat_info_v = get(reg_compat_info, pkg.version, nothing) - versions = keys(reg_compat_info) - for v in versions - compat_info_v = get(reg_compat_info, v, nothing) - compat_info_v === nothing && continue - compat_info_v_uuid = compat_info_v[JULIA_UUID] - if VERSION in compat_info_v_uuid + info = Registry.registry_info(reg, reg_pkg) + # Check all versions for Julia compatibility (optimized: only fetch Julia compat) + for v in keys(info.version_info) + julia_vspec = Registry.query_compat_for_version(info, v, JULIA_UUID) + if julia_vspec !== nothing && VERSION in julia_vspec push!(julia_compatible_versions, v) end end @@ -3691,7 +3735,7 @@ function get_all_registered_versions( for reg in ctx.registries pkg = get(reg, uuid, nothing) if pkg !== nothing - info = Registry.registry_info(pkg) + info = Registry.registry_info(reg, pkg) union!(versions, keys(info.version_info)) end end diff --git a/src/Pkg.jl b/src/Pkg.jl index 1a992788c1..6a93ddd74c 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -110,8 +110,8 @@ include("GitTools.jl") include("PlatformEngines.jl") include("Versions.jl") include("Registry/Registry.jl") -include("Resolve/Resolve.jl") include("Types.jl") +include("Resolve/Resolve.jl") include("BinaryPlatformsCompat.jl") include("Artifacts.jl") const Artifacts = PkgArtifacts diff --git a/src/Registry/registry_instance.jl b/src/Registry/registry_instance.jl index d67f5419dc..28c07e27f9 100644 --- a/src/Registry/registry_instance.jl +++ b/src/Registry/registry_instance.jl @@ -36,10 +36,6 @@ custom_isfile(in_memory_registry::Union{Dict, Nothing}, folder::AbstractString, mutable struct VersionInfo const git_tree_sha1::Base.SHA1 const yanked::Bool - uncompressed_compat::Dict{UUID, VersionSpec} # lazily initialized - weak_uncompressed_compat::Dict{UUID, VersionSpec} # lazily initialized - - VersionInfo(git_tree_sha1::Base.SHA1, yanked::Bool) = new(git_tree_sha1, yanked) end # This is the information that exists in e.g. General/A/ACME @@ -54,221 +50,284 @@ struct PkgInfo # Versions.toml: version_info::Dict{VersionNumber, VersionInfo} - # Compat.toml - compat::Dict{VersionRange, Dict{String, VersionSpec}} - - # Deps.toml - deps::Dict{VersionRange, Dict{String, UUID}} + # Deps.toml - which dependencies exist + deps::Dict{VersionRange, Set{UUID}} - # WeakCompat.toml - weak_compat::Dict{VersionRange, Dict{String, VersionSpec}} + # Compat.toml - version constraints on deps + compat::Dict{VersionRange, Dict{UUID, VersionSpec}} - # WeakDeps.toml - weak_deps::Dict{VersionRange, Dict{String, UUID}} + # WeakDeps.toml - which weak dependencies exist + weak_deps::Dict{VersionRange, Set{UUID}} - info_lock::ReentrantLock + # WeakCompat.toml - version constraints on weak deps + weak_compat::Dict{VersionRange, Dict{UUID, VersionSpec}} end isyanked(pkg::PkgInfo, v::VersionNumber) = pkg.version_info[v].yanked treehash(pkg::PkgInfo, v::VersionNumber) = pkg.version_info[v].git_tree_sha1 isdeprecated(pkg::PkgInfo) = pkg.deprecated !== nothing -function uncompress(compressed::Dict{VersionRange, Dict{String, T}}, vsorted::Vector{VersionNumber}) where {T} - @assert issorted(vsorted) - uncompressed = Dict{VersionNumber, Dict{String, T}}() - for v in vsorted - uncompressed[v] = Dict{String, T}() - end - for (vs, data) in compressed - first = length(vsorted) + 1 - # We find the first and last version that are in the range - # and since the versions are sorted, all versions in between are sorted - for i in eachindex(vsorted) - v = vsorted[i] - v in vs && (first = i; break) - end - last = 0 - for i in reverse(eachindex(vsorted)) - v = vsorted[i] - v in vs && (last = i; break) - end - for i in first:last - v = vsorted[i] - uv = uncompressed[v] - for (key, value) in data - if haskey(uv, key) - error("Overlapping ranges for $(key) for version $v in registry.") - else - uv[key] = value - end - end - end - end - return uncompressed -end - const JULIA_UUID = UUID("1222c4b2-2114-5bfd-aeef-88e4692bbb3e") -function initialize_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info)) - # Only valid to call this with existing versions of the package - # Remove all versions we have already uncompressed - versions = filter!(v -> !isdefined(pkg.version_info[v], :uncompressed_compat), collect(versions)) - - sort!(versions) - - uncompressed_compat = uncompress(pkg.compat, versions) - uncompressed_deps = uncompress(pkg.deps, versions) - for v in versions - vinfo = pkg.version_info[v] - compat = Dict{UUID, VersionSpec}() - uncompressed_deps_v = uncompressed_deps[v] - # Everything depends on Julia - uncompressed_deps_v["julia"] = JULIA_UUID - uncompressed_compat_v = uncompressed_compat[v] - for (pkg, uuid) in uncompressed_deps_v - vspec = get(uncompressed_compat_v, pkg, nothing) - compat[uuid] = vspec === nothing ? VersionSpec() : vspec - end - @assert !isdefined(vinfo, :uncompressed_compat) - vinfo.uncompressed_compat = compat - end - return pkg -end -function initialize_weak_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info)) - # Only valid to call this with existing versions of the package - # Remove all versions we have already uncompressed - versions = filter!(v -> !isdefined(pkg.version_info[v], :weak_uncompressed_compat), collect(versions)) +mutable struct PkgEntry + # Registry.toml: + const path::String + const registry_path::String + const name::String + const uuid::UUID - sort!(versions) + # Version.toml / (Compat.toml / Deps.toml): + info::PkgInfo # lazily initialized - weak_uncompressed_compat = uncompress(pkg.weak_compat, versions) - weak_uncompressed_deps = uncompress(pkg.weak_deps, versions) + PkgEntry(path, registry_path, name, uuid) = new(path, registry_path, name, uuid #= undef =#) +end - for v in versions - vinfo = pkg.version_info[v] - weak_compat = Dict{UUID, VersionSpec}() - weak_uncompressed_deps_v = weak_uncompressed_deps[v] - weak_uncompressed_compat_v = weak_uncompressed_compat[v] - for (pkg, uuid) in weak_uncompressed_deps_v - vspec = get(weak_uncompressed_compat_v, pkg, nothing) - weak_compat[uuid] = vspec === nothing ? VersionSpec() : vspec +# Helper to load deps data from Deps.toml or WeakDeps.toml +# Returns Dict{VersionRange, Set{UUID}} - just lists which deps exist +function load_deps_data(in_memory_registry, registry_path, pkg_path, filename, name_to_uuid) + deps_data_toml = custom_isfile(in_memory_registry, registry_path, joinpath(pkg_path, filename)) ? + parsefile(in_memory_registry, registry_path, joinpath(pkg_path, filename)) : Dict{String, Any}() + deps = Dict{VersionRange, Set{UUID}}() + for (v, data) in deps_data_toml + data = data::Dict{String, Any} + vr = VersionRange(v) + d = Set{UUID}() + for (dep, uuid_str) in data + uuid_val = UUID(uuid_str::String) + push!(d, uuid_val) + name_to_uuid[dep] = uuid_val end - @assert !isdefined(vinfo, :weak_uncompressed_compat) - vinfo.weak_uncompressed_compat = weak_compat + deps[vr] = d end - return pkg + return deps end -function compat_info(pkg::PkgInfo) - @lock pkg.info_lock initialize_uncompressed!(pkg) - return Dict(v => info.uncompressed_compat for (v, info) in pkg.version_info) +# Helper to load compat data from Compat.toml or WeakCompat.toml +function load_compat_data(in_memory_registry, registry_path, pkg_path, filename, name_to_uuid) + compat_data_toml = custom_isfile(in_memory_registry, registry_path, joinpath(pkg_path, filename)) ? + parsefile(in_memory_registry, registry_path, joinpath(pkg_path, filename)) : Dict{String, Any}() + compat = Dict{VersionRange, Dict{UUID, VersionSpec}}() + for (v, data) in compat_data_toml + data = data::Dict{String, Any} + vr = VersionRange(v) + d = Dict{UUID, VersionSpec}() + for (dep, vr_dep::Union{String, Vector{String}}) in data + d[name_to_uuid[dep]] = VersionSpec(vr_dep) + end + compat[vr] = d + end + return compat end -function weak_compat_info(pkg::PkgInfo) - if isempty(pkg.weak_deps) - return nothing +# Helper function to query just the dependencies (without compat specs) for a version +# Returns Set{UUID} of all dependencies (both strong and weak) for the given version +function query_deps_for_version( + deps_compressed::Dict{VersionRange, Set{UUID}}, + weak_deps_compressed::Dict{VersionRange, Set{UUID}}, + version::VersionNumber + )::Set{UUID} + result = Set{UUID}() + for compressed in (deps_compressed, weak_deps_compressed) + for (vrange, deps_set) in compressed + if version in vrange + union!(result, deps_set) + end + end end - @lock pkg.info_lock initialize_weak_uncompressed!(pkg) - return Dict(v => info.weak_uncompressed_compat for (v, info) in pkg.version_info) + return result end -mutable struct PkgEntry - # Registry.toml: - const path::String - const registry_path::String - const name::String - const uuid::UUID - - const in_memory_registry::Union{Dict{String, String}, Nothing} - # Lock for thread-safe lazy loading - const info_lock::ReentrantLock - # Version.toml / (Compat.toml / Deps.toml): - info::PkgInfo # lazily initialized +# Helper function to query deps for a specific version from multi-registry maps +function query_deps_for_version( + deps_map::Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}}, + weak_deps_map::Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}}, + uuid::UUID, + version::VersionNumber + )::Set{UUID} + result = Set{UUID}() + deps_list = get(Vector{Dict{VersionRange, Set{UUID}}}, deps_map, uuid) + weak_deps_list = get(Vector{Dict{VersionRange, Set{UUID}}}, weak_deps_map, uuid) + + # Query each registry's data + for i in eachindex(deps_list) + deps_compressed = deps_list[i] + weak_deps_compressed = weak_deps_list[i] + union!(result, query_deps_for_version(deps_compressed, weak_deps_compressed, version)) + end - PkgEntry(path, registry_path, name, uuid, in_memory_registry) = new(path, registry_path, name, uuid, in_memory_registry, ReentrantLock() #= undef =#) + return result end -registry_info(pkg::PkgEntry) = init_package_info!(pkg) +# Helper function to query compressed compat data from PkgInfo +# Convenience wrapper that uses PkgInfo's compressed data directly +# Returns Dict{UUID, VersionSpec} if target_uuid is nothing +# Returns Union{VersionSpec, Nothing} if target_uuid is provided +function query_compat_for_version( + pkg_info::PkgInfo, + version::VersionNumber, + target_uuid::Union{UUID, Nothing} = nothing + ) + return query_compat_for_version(pkg_info.deps, pkg_info.compat, pkg_info.weak_deps, pkg_info.weak_compat, version, target_uuid) +end -function init_package_info!(pkg::PkgEntry) - # Thread-safe lazy loading with double-check pattern - return @lock pkg.info_lock begin - # Double-check: if another thread loaded while we were waiting for the lock - isdefined(pkg, :info) && return pkg.info +# Mutating helper function to query compressed compat data for a specific version +# Merges deps (which dependencies exist) with compat (version constraints on those deps) +# Dependencies without explicit compat entries get VersionSpec() (any version) +# Includes both strong and weak dependencies +# If target_uuid is provided, only includes that UUID if it exists +# The result dictionary is emptied before populating +function query_compat_for_version!( + result::Dict{UUID, VersionSpec}, + deps_compressed::Dict{VersionRange, Set{UUID}}, + compat_compressed::Dict{VersionRange, Dict{UUID, VersionSpec}}, + weak_deps_compressed::Dict{VersionRange, Set{UUID}}, + weak_compat_compressed::Dict{VersionRange, Dict{UUID, VersionSpec}}, + version::VersionNumber, + target_uuid::Union{UUID, Nothing} = nothing + ) + empty!(result) + + for deps_dict in (deps_compressed, weak_deps_compressed) + for (vrange, deps_set) in deps_dict + if version in vrange + for dep_uuid in deps_set + if target_uuid === nothing || dep_uuid == target_uuid + result[dep_uuid] = VersionSpec() # Default: any version + end + end + end + end + end - path = pkg.registry_path + # Override with explicit compat specs from regular and weak compat + for compat_dict in (compat_compressed, weak_compat_compressed) + for (vrange, compat_entries) in compat_dict + if version in vrange + for (dep_uuid, vspec) in compat_entries + if target_uuid === nothing || dep_uuid == target_uuid + result[dep_uuid] = vspec + end + end + end + end + end - d_p = parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Package.toml")) - name = d_p["name"]::String - name != pkg.name && error("inconsistent name in Registry.toml ($(name)) and Package.toml ($(pkg.name)) for pkg at $(path)") - repo = get(d_p, "repo", nothing)::Union{Nothing, String} - subdir = get(d_p, "subdir", nothing)::Union{Nothing, String} + return nothing +end - # The presence of a [metadata.deprecated] table indicates the package is deprecated - # We store the raw table to allow other tools to use the metadata - metadata = get(d_p, "metadata", nothing)::Union{Nothing, Dict{String, Any}} - deprecated = metadata !== nothing ? get(metadata, "deprecated", nothing)::Union{Nothing, Dict{String, Any}} : nothing +# Non-mutating wrapper for backwards compatibility +# If target_uuid is provided, returns VersionSpec or nothing for that specific UUID +# If target_uuid is nothing, returns Dict{UUID, VersionSpec} for all dependencies +function query_compat_for_version( + deps_compressed::Dict{VersionRange, Set{UUID}}, + compat_compressed::Dict{VersionRange, Dict{UUID, VersionSpec}}, + weak_deps_compressed::Dict{VersionRange, Set{UUID}}, + weak_compat_compressed::Dict{VersionRange, Dict{UUID, VersionSpec}}, + version::VersionNumber, + target_uuid::Union{UUID, Nothing} = nothing + ) + result = Dict{UUID, VersionSpec}() + query_compat_for_version!(result, deps_compressed, compat_compressed, weak_deps_compressed, weak_compat_compressed, version, target_uuid) + + # If a specific UUID was requested, return just its VersionSpec (or nothing) + if target_uuid !== nothing + return get(result, target_uuid, nothing) + end - # Versions.toml - d_v = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) ? - parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) : Dict{String, Any}() - version_info = Dict{VersionNumber, VersionInfo}( - VersionNumber(k) => - VersionInfo(SHA1(v["git-tree-sha1"]::String), get(v, "yanked", false)::Bool) for (k, v) in d_v - ) + return result +end - # Compat.toml - compat_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Compat.toml")) ? - parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Compat.toml")) : Dict{String, Any}() - compat = Dict{VersionRange, Dict{String, VersionSpec}}() - for (v, data) in compat_data_toml - data = data::Dict{String, Any} - vr = VersionRange(v) - d = Dict{String, VersionSpec}(dep => VersionSpec(vr_dep) for (dep, vr_dep::Union{String, Vector{String}}) in data) - compat[vr] = d +# Helper to check if a UUID is in the weak deps for a specific version +function is_weak_dep( + weak_compressed::Dict{VersionRange, Set{UUID}}, + version::VersionNumber, + dep_uuid::UUID + )::Bool + for (vrange, weak_set) in weak_compressed + if version in vrange && (dep_uuid in weak_set) + return true end + end + return false +end - # Deps.toml - deps_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Deps.toml")) ? - parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Deps.toml")) : Dict{String, Any}() - deps = Dict{VersionRange, Dict{String, UUID}}() - for (v, data) in deps_data_toml - data = data::Dict{String, Any} - vr = VersionRange(v) - d = Dict{String, UUID}(dep => UUID(uuid) for (dep, uuid::String) in data) - deps[vr] = d - end - # All packages depend on julia - deps[VersionRange()] = Dict("julia" => JULIA_UUID) - - # WeakCompat.toml - weak_compat_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) ? - parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) : Dict{String, Any}() - weak_compat = Dict{VersionRange, Dict{String, VersionSpec}}() - for (v, data) in weak_compat_data_toml - data = data::Dict{String, Any} - vr = VersionRange(v) - d = Dict{String, VersionSpec}(dep => VersionSpec(vr_dep) for (dep, vr_dep::Union{String, Vector{String}}) in data) - weak_compat[vr] = d +# Helper function to query compat across multiple registries +# Each registry has its own compressed dictionaries and version set +# Only queries a registry if the version actually exists in that registry +function query_compat_for_version_multi_registry!( + result::Dict{UUID, VersionSpec}, + reg_result::Dict{UUID, VersionSpec}, + deps_list::Vector{Dict{VersionRange, Set{UUID}}}, + compat_list::Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}, + weak_deps_list::Vector{Dict{VersionRange, Set{UUID}}}, + weak_compat_list::Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}, + versions_per_registry::Vector{Set{VersionNumber}}, + version::VersionNumber + ) + empty!(result) + + # Query each registry's data separately + for i in eachindex(deps_list) + # CRITICAL: Only query this registry if the version exists in it! + if !(version in versions_per_registry[i]) + continue end - # WeakDeps.toml - weak_deps_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakDeps.toml")) ? - parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakDeps.toml")) : Dict{String, Any}() - weak_deps = Dict{VersionRange, Dict{String, UUID}}() - for (v, data) in weak_deps_data_toml - data = data::Dict{String, Any} - vr = VersionRange(v) - d = Dict{String, UUID}(dep => UUID(uuid) for (dep, uuid::String) in data) - weak_deps[vr] = d + reg_deps = deps_list[i] + reg_compat = compat_list[i] + reg_weak_deps = weak_deps_list[i] + reg_weak_compat = weak_compat_list[i] + + # Use the mutating query function to avoid allocation + query_compat_for_version!(reg_result, reg_deps, reg_compat, reg_weak_deps, reg_weak_compat, version) + + # Merge results, preferring the first registry's compat if there's overlap + for (uuid, vspec) in reg_result + if !haskey(result, uuid) + result[uuid] = vspec + end + # If uuid already exists, keep the first registry's vspec (first wins) end + end - @assert !isdefined(pkg, :info) - pkg.info = PkgInfo(repo, subdir, deprecated, version_info, compat, deps, weak_compat, weak_deps, pkg.info_lock) + return nothing +end - return pkg.info +# Validate that no version ranges overlap for the same dependency +# This enforces the registry invariant that each dependency should be specified +# at most once for any given version +# Works with any collection type (Set, Dict, etc.) and any key type (UUID, String, etc.) +function validate_no_overlapping_ranges( + compressed::Dict{VersionRange, T}, + versions::Vector{VersionNumber}, + pkg_name::String, + data_type::String, # "Deps", "WeakDeps", "Compat", or "WeakCompat" + name_to_uuid::Dict{String, UUID} + ) where {T} + # Build inverse mapping for better error messages + uuid_to_name = Dict{UUID, String}(uuid => name for (name, uuid) in name_to_uuid) + + # For each version, check that no dependency UUID appears in multiple ranges + for v in versions + seen_deps = Dict{UUID, VersionRange}() + for (vrange, dep_collection) in compressed + if v in vrange + # Works for both Set{UUID} (iterate directly) and Dict{UUID,...} (iterate keys) + for dep_uuid in (dep_collection isa AbstractDict ? keys(dep_collection) : dep_collection) + if haskey(seen_deps, dep_uuid) + dep_name = get(uuid_to_name, dep_uuid, string(dep_uuid)) + error( + "Overlapping ranges for dependency $(dep_name) in $(pkg_name) $(data_type).toml: " * + "version $v is covered by both $(seen_deps[dep_uuid]) and $(vrange)" + ) + end + seen_deps[dep_uuid] = vrange + end + end + end end + return end @@ -324,6 +383,86 @@ end const REGISTRY_CACHE = Dict{String, Tuple{Base.SHA1, Bool, RegistryInstance}}() +function init_package_info!(registry::RegistryInstance, pkg::PkgEntry) + # Thread-safe lazy loading with double-check pattern + # Use the registry's load_lock to protect lazy loading of package info + return @lock registry.load_lock begin + # Double-check: if another thread loaded while we were waiting for the lock + isdefined(pkg, :info) && return pkg.info + + path = pkg.registry_path + in_memory_registry = registry.in_memory_registry + + d_p = parsefile(in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Package.toml")) + name = d_p["name"]::String + name != pkg.name && error("inconsistent name in Registry.toml ($(name)) and Package.toml ($(pkg.name)) for pkg at $(path)") + repo = get(d_p, "repo", nothing)::Union{Nothing, String} + subdir = get(d_p, "subdir", nothing)::Union{Nothing, String} + + # The presence of a [metadata.deprecated] table indicates the package is deprecated + # We store the raw table to allow other tools to use the metadata + metadata = get(d_p, "metadata", nothing)::Union{Nothing, Dict{String, Any}} + deprecated = metadata !== nothing ? get(metadata, "deprecated", nothing)::Union{Nothing, Dict{String, Any}} : nothing + + # Versions.toml + d_v = custom_isfile(in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) ? + parsefile(in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) : Dict{String, Any}() + version_info = Dict{VersionNumber, VersionInfo}( + VersionNumber(k) => + VersionInfo(SHA1(v["git-tree-sha1"]::String), get(v, "yanked", false)::Bool) for (k, v) in d_v + ) + + # Deps.toml (load first to build name -> UUID mapping) + name_to_uuid = Dict{String, UUID}() + deps = load_deps_data(in_memory_registry, pkg.registry_path, pkg.path, "Deps.toml", name_to_uuid) + # All packages depend on julia + deps[VersionRange()] = Set([JULIA_UUID]) + name_to_uuid["julia"] = JULIA_UUID + + # WeakDeps.toml (load to extend name -> UUID mapping) + weak_deps = load_deps_data(in_memory_registry, pkg.registry_path, pkg.path, "WeakDeps.toml", name_to_uuid) + + # Compat.toml (convert names to UUIDs using the mapping) + compat = load_compat_data(in_memory_registry, pkg.registry_path, pkg.path, "Compat.toml", name_to_uuid) + + # WeakCompat.toml (convert names to UUIDs using the mapping) + weak_compat = load_compat_data(in_memory_registry, pkg.registry_path, pkg.path, "WeakCompat.toml", name_to_uuid) + + #= + # These validations are a bit too expensive + # RegistryTools does this already: https://github.com/JuliaRegistries/RegistryTools.jl/blob/b5ff4d541b0aad2261ac21416113cee9718e28b3/src/Compress.jl#L64 + # Validate that no ranges overlap for the same dependency (registry invariant) + versions_list = sort!(collect(keys(version_info))) + if !isempty(deps) + validate_no_overlapping_ranges(deps, versions_list, pkg.name, "Deps", name_to_uuid) + end + if !isempty(weak_deps) + validate_no_overlapping_ranges(weak_deps, versions_list, pkg.name, "WeakDeps", name_to_uuid) + end + if !isempty(compat) + validate_no_overlapping_ranges(compat, versions_list, pkg.name, "Compat", name_to_uuid) + end + if !isempty(weak_compat) + validate_no_overlapping_ranges(weak_compat, versions_list, pkg.name, "WeakCompat", name_to_uuid) + end + =# + + @assert !isdefined(pkg, :info) + pkg.info = PkgInfo(repo, subdir, deprecated, version_info, deps, compat, weak_deps, weak_compat) + + # Free memory: delete the package's files from in_memory_registry since we've fully parsed them + if in_memory_registry !== nothing + for filename in ("Package.toml", "Versions.toml", "Deps.toml", "WeakDeps.toml", "Compat.toml", "WeakCompat.toml") + delete!(in_memory_registry, to_tar_path_format(joinpath(pkg.path, filename))) + end + end + + return pkg.info + end +end + +registry_info(registry::RegistryInstance, pkg::PkgEntry) = init_package_info!(registry, pkg) + @noinline function _ensure_registry_loaded_slow!(r::RegistryInstance) return @lock r.load_lock begin # Double-check pattern: if another thread loaded while we were waiting for the lock @@ -347,7 +486,7 @@ const REGISTRY_CACHE = Dict{String, Tuple{Base.SHA1, Bool, RegistryInstance}}() info::Dict{String, Any} name = info["name"]::String pkgpath = info["path"]::String - pkg = PkgEntry(pkgpath, getfield(r, :path), name, uuid, r.in_memory_registry) + pkg = PkgEntry(pkgpath, getfield(r, :path), name, uuid) r.pkgs[uuid] = pkg end diff --git a/src/Resolve/Resolve.jl b/src/Resolve/Resolve.jl index 93d0f036bd..bb45407618 100644 --- a/src/Resolve/Resolve.jl +++ b/src/Resolve/Resolve.jl @@ -3,6 +3,8 @@ module Resolve using ..Versions +using ..Registry +using ..Types import ..stdout_f, ..stderr_f using Printf diff --git a/src/Resolve/graphtype.jl b/src/Resolve/graphtype.jl index abc5b2abfb..9e4b1b6425 100644 --- a/src/Resolve/graphtype.jl +++ b/src/Resolve/graphtype.jl @@ -113,19 +113,20 @@ mutable struct GraphData rlog::ResolveLog function GraphData( - compat::Dict{UUID, Dict{VersionNumber, Dict{UUID, VersionSpec}}}, + compat_compressed::Dict{UUID, Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}}, + pkg_versions::Dict{UUID, Vector{VersionNumber}}, uuid_to_name::Dict{UUID, String}, verbose::Bool = false ) # generate pkgs - pkgs = sort!(collect(keys(compat))) + pkgs = sort!(collect(keys(pkg_versions))) np = length(pkgs) # generate pdict pdict = Dict{UUID, Int}(pkgs[p0] => p0 for p0 in 1:np) - # generate spp and pvers - pvers = [sort!(collect(keys(compat[pkgs[p0]]))) for p0 in 1:np] + # generate spp and pvers from provided version lists + pvers = [pkg_versions[pkgs[p0]] for p0 in 1:np] spp = length.(pvers) .+ 1 # generate vdict @@ -143,7 +144,7 @@ mutable struct GraphData d = Dict{InstState, Set{InstState}}() for v0 in 1:spp[p0] let p0 = p0 # Due to https://github.com/JuliaLang/julia/issues/15276 - d[eq_vn(v0, p0)] = Set([eq_vn(v0, p0)]) + d[eq_vn(v0, p0)] = Set{InstState}((eq_vn(v0, p0),)) end end eq_classes[pkgs[p0]] = d @@ -235,8 +236,12 @@ mutable struct Graph cavfld::Vector{FieldValue} function Graph( - compat::Dict{UUID, Dict{VersionNumber, Dict{UUID, VersionSpec}}}, - compat_weak::Dict{UUID, Dict{VersionNumber, Set{UUID}}}, + deps_compressed::Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}}, + compat_compressed::Dict{UUID, Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}}, + weak_deps_compressed::Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}}, + weak_compat_compressed::Dict{UUID, Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}}, + pkg_versions::Dict{UUID, Vector{VersionNumber}}, + pkg_versions_per_registry::Dict{UUID, Vector{Set{VersionNumber}}}, uuid_to_name::Dict{UUID, String}, reqs::Requires, fixed::Dict{UUID, Fixed}, @@ -248,49 +253,93 @@ mutable struct Graph uuid_to_name[uuid_julia] = "julia" if julia_version !== nothing fixed[uuid_julia] = Fixed(julia_version) - compat[uuid_julia] = Dict(julia_version => Dict{VersionNumber, Dict{UUID, VersionSpec}}()) + deps_compressed[uuid_julia] = [Dict{VersionRange, Set{UUID}}()] + compat_compressed[uuid_julia] = [Dict{VersionRange, Dict{UUID, VersionSpec}}()] + weak_deps_compressed[uuid_julia] = [Dict{VersionRange, Set{UUID}}()] + weak_compat_compressed[uuid_julia] = [Dict{VersionRange, Dict{UUID, VersionSpec}}()] + pkg_versions[uuid_julia] = [julia_version] + pkg_versions_per_registry[uuid_julia] = [Set([julia_version])] else - compat[uuid_julia] = Dict{VersionNumber, Dict{UUID, VersionSpec}}() + deps_compressed[uuid_julia] = [Dict{VersionRange, Set{UUID}}()] + compat_compressed[uuid_julia] = [Dict{VersionRange, Dict{UUID, VersionSpec}}()] + weak_deps_compressed[uuid_julia] = [Dict{VersionRange, Set{UUID}}()] + weak_compat_compressed[uuid_julia] = [Dict{VersionRange, Dict{UUID, VersionSpec}}()] + pkg_versions[uuid_julia] = VersionNumber[] + pkg_versions_per_registry[uuid_julia] = [Set{VersionNumber}()] end - data = GraphData(compat, uuid_to_name, verbose) + data = GraphData(compat_compressed, pkg_versions, uuid_to_name, verbose) pkgs, np, spp, pdict, pvers, vdict, rlog = data.pkgs, data.np, data.spp, data.pdict, data.pvers, data.vdict, data.rlog extended_deps = let spp = spp # Due to https://github.com/JuliaLang/julia/issues/15276 [Vector{Dict{Int, BitVector}}(undef, spp[p0] - 1) for p0 in 1:np] end - for p0 in 1:np, v0 in 1:(spp[p0] - 1) - vn = pvers[p0][v0] - req = Dict{Int, VersionSpec}() + vnmap = Dict{UUID, VersionSpec}() + reg_result = Dict{UUID, VersionSpec}() + req = Dict{Int, VersionSpec}() + for p0 in 1:np uuid0 = pkgs[p0] - vnmap = get(Dict{UUID, VersionSpec}, compat[uuid0], vn) - for (uuid1, vs) in vnmap - p1 = pdict[uuid1] - p1 == p0 && error("Package $(pkgID(pkgs[p0], uuid_to_name)) version $vn has a dependency with itself") - # check conflicts instead of intersecting? - # (intersecting is used by fixed packages though...) - req_p1 = get(req, p1, nothing) - if req_p1 == nothing - req[p1] = vs - else - req[p1] = req_p1 ∩ vs + + # Query compressed deps and compat data for this version (including weak deps) + # We have a vector of per-registry dictionaries, need to query across all + uuid0_deps_list = get(Vector{Dict{VersionRange, Set{UUID}}}, deps_compressed, uuid0) + uuid0_compat_list = get(Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}, compat_compressed, uuid0) + uuid0_weak_deps_list = get(Vector{Dict{VersionRange, Set{UUID}}}, weak_deps_compressed, uuid0) + uuid0_weak_compat_list = get(Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}, weak_compat_compressed, uuid0) + uuid0_versions_per_reg = get(Vector{Set{VersionNumber}}, pkg_versions_per_registry, uuid0) + + for v0 in 1:(spp[p0] - 1) + vn = pvers[p0][v0] + empty!(req) + Registry.query_compat_for_version_multi_registry!(vnmap, reg_result, uuid0_deps_list, uuid0_compat_list, uuid0_weak_deps_list, uuid0_weak_compat_list, uuid0_versions_per_reg, vn) + + # Filter out incompatible stdlib compat entries from registry dependencies + for (dep_uuid, dep_compat) in vnmap + if Types.is_stdlib(dep_uuid) && !(dep_uuid in Types.UPGRADABLE_STDLIBS_UUIDS) + stdlib_ver = Types.stdlib_version(dep_uuid, julia_version) + if stdlib_ver !== nothing && !isempty(dep_compat) && !(stdlib_ver in dep_compat) + @debug "Ignoring incompatible stdlib compat entry" dep = get(uuid_to_name, dep_uuid, string(dep_uuid)) stdlib_ver dep_compat package = uuid_to_name[uuid0] version = vn + delete!(vnmap, dep_uuid) + end + end end - end - # Translate the requirements into bit masks - # Hot code, measure performance before changing - req_msk = Dict{Int, BitVector}() - sizehint!(req_msk, length(req)) - maybe_weak = haskey(compat_weak, uuid0) && haskey(compat_weak[uuid0], vn) - for (p1, vs) in req - pv = pvers[p1] - req_msk_p1 = BitVector(undef, spp[p1]) - @inbounds for i in 1:(spp[p1] - 1) - req_msk_p1[i] = pv[i] ∈ vs + + for (uuid1, vs) in vnmap + p1 = pdict[uuid1] + p1 == p0 && error("Package $(pkgID(pkgs[p0], uuid_to_name)) version $vn has a dependency with itself") + # check conflicts instead of intersecting? + # (intersecting is used by fixed packages though...) + req_p1 = get(req, p1, nothing) + if req_p1 == nothing + req[p1] = vs + else + req[p1] = req_p1 ∩ vs + end + end + + # Translate the requirements into bit masks + # Hot code, measure performance before changing + req_msk = Dict{Int, BitVector}() + sizehint!(req_msk, length(req)) + + for (p1, vs) in req + pv = pvers[p1] + # Allocate BitVector with space for weak dep flag + req_msk_p1 = BitVector(undef, spp[p1]) + # Use optimized batch version check (fills indices 1 through spp[p1]-1) + Versions.matches_spec_range!(req_msk_p1, pv, vs, spp[p1] - 1) + # Check if this is a weak dep across all registries + weak = false + for weak_deps_dict in uuid0_weak_deps_list + if Registry.is_weak_dep(weak_deps_dict, vn, pkgs[p1]) + weak = true + break + end + end + req_msk_p1[end] = weak + req_msk[p1] = req_msk_p1 end - weak = maybe_weak && (pkgs[p1] ∈ compat_weak[uuid0][vn]) - req_msk_p1[end] = weak - req_msk[p1] = req_msk_p1 + extended_deps[p0][v0] = req_msk end - extended_deps[p0][v0] = req_msk end gadj = [Int[] for p0 in 1:np] @@ -327,7 +376,7 @@ mutable struct Graph bmt = gmsk[p1][j1] end - for v1 in 1:spp[p1] + @inbounds for v1 in 1:spp[p1] rmsk1[v1] && continue bm[v1, v0] = false bmt[v0, v1] = false diff --git a/src/Types.jl b/src/Types.jl index ecd1c29ed7..4c631f6faa 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -937,7 +937,7 @@ function set_repo_source_from_registry!(ctx, pkg) for reg in ctx.registries regpkg = get(reg, pkg.uuid, nothing) regpkg === nothing && continue - info = Pkg.Registry.registry_info(regpkg) + info = Pkg.Registry.registry_info(reg, regpkg) url = info.repo url === nothing && continue pkg.repo.source = url @@ -1343,7 +1343,7 @@ function registered_uuid(registries::Vector{Registry.RegistryInstance}, name::St for reg in registries pkg = get(reg, uuid, nothing) pkg === nothing && continue - info = Pkg.Registry.registry_info(pkg) + info = Pkg.Registry.registry_info(reg, pkg) repo = info.repo repo === nothing && continue push!(repo_infos, (reg.name, repo, uuid)) diff --git a/src/Versions.jl b/src/Versions.jl index a572e198fa..0d7b419400 100644 --- a/src/Versions.jl +++ b/src/Versions.jl @@ -237,6 +237,38 @@ function Base.in(v::VersionNumber, s::VersionSpec) return false end +# Optimized batch version check for version lists +# Fills dest[1:n] indicating which versions are in the VersionSpec +# Optimized for sorted version lists (but works correctly even if unsorted) +# Note: Only fills indices 1:n, leaves rest of dest unchanged +function matches_spec_range!(dest::BitVector, versions::AbstractVector{VersionNumber}, spec::VersionSpec, n::Int) + @assert length(versions) == n + @assert length(dest) >= n + + # Initialize to false + dest[1:n] .= false + + isempty(spec.ranges) && return dest + + # Assumes versions are sorted (as created in Operations.jl:1002) + # If sorted, this avoids O(n*m) comparisons by scanning linearly + @inbounds for range in spec.ranges + # Find first version that could be in range + i = 1 + while i <= n && !(range.lower ≲ versions[i]) + i += 1 + end + + # Mark all versions in range + while i <= n && versions[i] ≲ range.upper + dest[i] = true + i += 1 + end + end + + return dest +end + Base.copy(vs::VersionSpec) = VersionSpec(vs) const empty_versionspec = VersionSpec(VersionRange[]) diff --git a/test/registry.jl b/test/registry.jl index d81733f94b..352f4f3666 100644 --- a/test/registry.jl +++ b/test/registry.jl @@ -397,7 +397,7 @@ end @test haskey(reg, pkg_uuid) pkg_entry = reg[pkg_uuid] - pkg_info = Pkg.Registry.registry_info(pkg_entry) + pkg_info = Pkg.Registry.registry_info(reg, pkg_entry) # Test that deprecated info is loaded correctly @test Pkg.Registry.isdeprecated(pkg_info) @@ -408,7 +408,7 @@ end # Test that non-deprecated package is not marked as deprecated example1_uuid = UUID("c5f1542f-b8aa-45da-ab42-05303d706c66") example1_entry = reg[example1_uuid] - example1_info = Pkg.Registry.registry_info(example1_entry) + example1_info = Pkg.Registry.registry_info(reg, example1_entry) @test !Pkg.Registry.isdeprecated(example1_info) @test example1_info.deprecated === nothing diff --git a/test/resolve_utils.jl b/test/resolve_utils.jl index f0a152b56f..4168f00c41 100644 --- a/test/resolve_utils.jl +++ b/test/resolve_utils.jl @@ -40,8 +40,6 @@ function graph_from_data(deps_data) uuid_to_name = Dict{UUID, String}() uuid(p) = storeuuid(p, uuid_to_name) fixed = Dict{UUID, Fixed}() - all_compat = Dict{UUID, Dict{VersionNumber, Dict{UUID, VersionSpec}}}() - all_compat_w = Dict{UUID, Dict{VersionNumber, Set{UUID}}}() deps = Dict{String, Dict{VersionNumber, Dict{String, VersionSpec}}}() deps_w = Dict{String, Dict{VersionNumber, Set{String}}}() @@ -63,26 +61,76 @@ function graph_from_data(deps_data) push!(get!(Set{String}, get!(Dict{VersionNumber, Set{String}}, deps_w, p), vn), rp) end end + # Build pkg_versions map + pkg_versions = Dict{UUID, Vector{VersionNumber}}() for (p, preq) in deps u = uuid(p) - deps_pkgs = Dict{String, Set{VersionNumber}}() - for (vn, vreq) in deps[p], rp in keys(vreq) - push!(get!(Set{VersionNumber}, deps_pkgs, rp), vn) - end - all_compat[u] = Dict{VersionNumber, Dict{UUID, VersionSpec}}() + pkg_versions[u] = sort!(collect(keys(preq))) + end + + # Convert per-version data to compressed format + # For tests, each version gets its own VersionRange + all_deps_compressed = Dict{UUID, Dict{VersionRange, Set{UUID}}}() + all_compat_compressed = Dict{UUID, Dict{VersionRange, Dict{UUID, VersionSpec}}}() + all_weak_deps_compressed = Dict{UUID, Dict{VersionRange, Set{UUID}}}() + all_weak_compat_compressed = Dict{UUID, Dict{VersionRange, Dict{UUID, VersionSpec}}}() + + for (p, preq) in deps + u = uuid(p) + all_deps_compressed[u] = Dict{VersionRange, Set{UUID}}() + all_compat_compressed[u] = Dict{VersionRange, Dict{UUID, VersionSpec}}() + all_weak_deps_compressed[u] = Dict{VersionRange, Set{UUID}}() + all_weak_compat_compressed[u] = Dict{VersionRange, Dict{UUID, VersionSpec}}() + for (vn, vreq) in preq - all_compat[u][vn] = Dict{UUID, VersionSpec}() + # Create a single-version range for this version + vrange = VersionRange(vn, vn) + deps_set = Set{UUID}() + compat_dict = Dict{UUID, VersionSpec}() + weak_deps_set = Set{UUID}() + weak_compat_dict = Dict{UUID, VersionSpec}() + for (rp, rvs) in vreq - all_compat[u][vn][uuid(rp)] = rvs + dep_uuid = uuid(rp) + push!(deps_set, dep_uuid) + compat_dict[dep_uuid] = rvs + # weak dependency? if haskey(deps_w, p) && haskey(deps_w[p], vn) && (rp ∈ deps_w[p][vn]) - # same as push!(all_compat_w[u][vn], uuid(rp)) but create keys as needed - push!(get!(Set{UUID}, get!(Dict{VersionNumber, Set{UUID}}, all_compat_w, u), vn), uuid(rp)) + push!(weak_deps_set, dep_uuid) + weak_compat_dict[dep_uuid] = rvs end end + + all_deps_compressed[u][vrange] = deps_set + all_compat_compressed[u][vrange] = compat_dict + if !isempty(weak_deps_set) + all_weak_deps_compressed[u][vrange] = weak_deps_set + all_weak_compat_compressed[u][vrange] = weak_compat_dict + end end end - return Graph(all_compat, all_compat_w, uuid_to_name, Requires(), fixed, VERBOSE) + + # Wrap in vectors for multi-registry support (tests simulate a single registry) + all_deps_compressed_vec = Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}}( + u => [d] for (u, d) in all_deps_compressed + ) + all_compat_compressed_vec = Dict{UUID, Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}}( + u => [c] for (u, c) in all_compat_compressed + ) + all_weak_deps_compressed_vec = Dict{UUID, Vector{Dict{VersionRange, Set{UUID}}}}( + u => [d] for (u, d) in all_weak_deps_compressed + ) + all_weak_compat_compressed_vec = Dict{UUID, Vector{Dict{VersionRange, Dict{UUID, VersionSpec}}}}( + u => [c] for (u, c) in all_weak_compat_compressed + ) + + # Create pkg_versions_per_registry (single registry with all versions) + pkg_versions_per_registry = Dict{UUID, Vector{Set{VersionNumber}}}( + u => [Set(versions)] for (u, versions) in pkg_versions + ) + + return Graph(all_deps_compressed_vec, all_compat_compressed_vec, all_weak_deps_compressed_vec, all_weak_compat_compressed_vec, pkg_versions, pkg_versions_per_registry, uuid_to_name, Requires(), fixed, VERBOSE) end function reqs_from_data(reqs_data, graph::Graph) reqs = Dict{UUID, VersionSpec}()