Move sorting from base to stdlib

NEWS.md

@@ -133,6 +133,9 @@ Standard library changes
 * An "IPython mode" which mimics the behaviour of the prompts and storing the evaluated result in `Out` can be
   activated with `REPL.ipython_mode!()`. See the manual for how to enable this at startup.
 
+#### Sort
+* `Sort` has moved out of base and is now a standard library. ([#46679])
+
 #### SparseArrays
 
 #### Test

base/Base.jl

@@ -346,10 +346,8 @@ include("ordering.jl")
 using .Order
 
 # Combinatorics
-include("sort.jl")
-using .Sort
 
-# BinaryPlatforms, used by Artifacts.  Needs `Sort`.
+# BinaryPlatforms, used by Artifacts.
 include("binaryplatforms.jl")
 
 # Fast math
@@ -465,8 +463,6 @@ for match = _methods(+, (Int, Int), -1, get_world_counter())
     Dict("abc" => Set())["abc"]
     pushfirst!([], sum)
     get(Base.pkgorigins, Base.PkgId(Base), nothing)
-    sort!([1,2,3])
-    unique!([1,2,3])
     cumsum([1,2,3])
     append!(Int[], BitSet())
     isempty(BitSet())
@@ -477,7 +473,7 @@ for match = _methods(+, (Int, Int), -1, get_world_counter())
     any(t->t[1].line > 1, [(LineNumberNode(2,:none), :(1+1))])
 
     # Code loading uses this
-    sortperm(mtime.(readdir(".")), rev=true)
+    mtime.(readdir(".", sort=false))
     # JLLWrappers uses these
     Dict{UUID,Set{String}}()[UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210")] = Set{String}()
     get!(Set{String}, Dict{UUID,Set{String}}(), UUID("692b3bcd-3c85-4b1f-b108-f13ce0eb3210"))

base/bitset.jl

@@ -430,4 +430,3 @@ end
 minimum(s::BitSet) = first(s)
 maximum(s::BitSet) = last(s)
 extrema(s::BitSet) = (first(s), last(s))
-issorted(s::BitSet) = true

base/compiler/compiler.jl

@@ -121,12 +121,10 @@ import Core.Compiler.CoreDocs
 Core.atdoc!(CoreDocs.docm)
 
 # sorting
-function sort! end
 function issorted end
 include("ordering.jl")
 using .Order
-include("sort.jl")
-using .Sort
+include("compiler/sort.jl")
 
 # We don't include some.jl, but this definition is still useful.
 something(x::Nothing, y...) = something(y...)

base/compiler/sort.jl

@@ -0,0 +1,100 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# reference on sorted binary search:
+#   http://www.tbray.org/ongoing/When/200x/2003/03/22/Binary
+
+# index of the first value of vector a that is greater than or equal to x;
+# returns lastindex(v)+1 if x is greater than all values in v.
+function searchsortedfirst(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::keytype(v) where T<:Integer
+    hi = hi + T(1)
+    len = hi - lo
+    @inbounds while len != 0
+        half_len = len >>> 0x01
+        m = lo + half_len
+        if lt(o, v[m], x)
+            lo = m + 1
+            len -= half_len + 1
+        else
+            hi = m
+            len = half_len
+        end
+    end
+    return lo
+end
+
+# index of the last value of vector a that is less than or equal to x;
+# returns firstindex(v)-1 if x is less than all values of v.
+function searchsortedlast(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::keytype(v) where T<:Integer
+    u = T(1)
+    lo = lo - u
+    hi = hi + u
+    @inbounds while lo < hi - u
+        m = midpoint(lo, hi)
+        if lt(o, x, v[m])
+            hi = m
+        else
+            lo = m
+        end
+    end
+    return lo
+end
+
+# returns the range of indices of v equal to x
+# if v does not contain x, returns a 0-length range
+# indicating the insertion point of x
+function searchsorted(v::AbstractVector, x, ilo::T, ihi::T, o::Ordering)::UnitRange{keytype(v)} where T<:Integer
+    u = T(1)
+    lo = ilo - u
+    hi = ihi + u
+    @inbounds while lo < hi - u
+        m = midpoint(lo, hi)
+        if lt(o, v[m], x)
+            lo = m
+        elseif lt(o, x, v[m])
+            hi = m
+        else
+            a = searchsortedfirst(v, x, max(lo,ilo), m, o)
+            b = searchsortedlast(v, x, m, min(hi,ihi), o)
+            return a : b
+        end
+    end
+    return (lo + 1) : (hi - 1)
+end
+
+for s in [:searchsortedfirst, :searchsortedlast, :searchsorted]
+    @eval begin
+        $s(v::AbstractVector, x, o::Ordering) = $s(v,x,firstindex(v),lastindex(v),o)
+        $s(v::AbstractVector, x;
+           lt=isless, by=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) =
+            $s(v,x,ord(lt,by,rev,order))
+    end
+end
+
+# An unstable sorting algorithm for internal use
+function sort!(v::Vector; by::Function=identity, (<)::Function=<)
+    isempty(v) && return v # This branch is hit 95% of the time
+
+    # Of the remaining 5%, this branch is hit less than 1% of the time
+    if length(v) > 200 # Heap sort prevents quadratic runtime
+        o = ord(<, by, true)
+        heapify!(v, o)
+        for i in lastindex(v):-1:2
+            y = v[i]
+            v[i] = v[1]
+            percolate_down!(v, 1, y, o, i-1)
+        end
+        return v
+    end
+
+    @inbounds for i in 2:length(v) # Insertion sort
+        x = v[i]
+        y = by(x)
+        while i > 1 && y < by(v[i-1])
+            v[i] = v[i-1]
+            i -= 1
+        end
+        v[i] = x
+    end
+
+    v
+end

base/compiler/ssair/ir.jl

@@ -534,16 +534,6 @@ end
 insert_node!(ir::IRCode, pos::Int, inst::NewInstruction, attach_after::Bool=false) =
     insert_node!(ir, SSAValue(pos), inst, attach_after)
 
-# For bootstrapping
-function my_sortperm(v)
-    p = Vector{Int}(undef, length(v))
-    for i = 1:length(v)
-        p[i] = i
-    end
-    sort!(p, Sort.DEFAULT_UNSTABLE, Order.Perm(Sort.Forward,v))
-    p
-end
-
 mutable struct IncrementalCompact
     ir::IRCode
     result::InstructionStream
@@ -560,9 +550,8 @@ mutable struct IncrementalCompact
     # This supports insertion while compacting
     new_new_nodes::NewNodeStream  # New nodes that were before the compaction point at insertion time
     new_new_used_ssas::Vector{Int}
-    # TODO: Switch these two to a min-heap of some sort
     pending_nodes::NewNodeStream  # New nodes that were after the compaction point at insertion time
-    pending_perm::Vector{Int}
+    pending_perm::Vector{Int} # pending_nodes.info[pending_perm] is in min-heap order by pos
 
     # State
     idx::Int
@@ -574,10 +563,9 @@ mutable struct IncrementalCompact
 
     function IncrementalCompact(code::IRCode, allow_cfg_transforms::Bool=false)
         # Sort by position with attach after nodes after regular ones
-        perm = my_sortperm(Int[let new_node = code.new_nodes.info[i]
-            (new_node.pos * 2 + Int(new_node.attach_after))
-            end for i in 1:length(code.new_nodes)])
-        new_len = length(code.stmts) + length(code.new_nodes)
+        info = code.new_nodes.info
+        perm = sort!(collect(eachindex(info)); by=i->(2info[i].pos+info[i].attach_after, i))
+        new_len = length(code.stmts) + length(info)
         result = InstructionStream(new_len)
         used_ssas = fill(0, new_len)
         new_new_used_ssas = Vector{Int}()
@@ -629,8 +617,9 @@ mutable struct IncrementalCompact
 
     # For inlining
     function IncrementalCompact(parent::IncrementalCompact, code::IRCode, result_offset)
-        perm = my_sortperm(Int[code.new_nodes.info[i].pos for i in 1:length(code.new_nodes)])
-        new_len = length(code.stmts) + length(code.new_nodes)
+        info = code.new_nodes.info
+        perm = sort!(collect(eachindex(info)); by=i->(info[i].pos, i))
+        new_len = length(code.stmts) + length(info)
         ssa_rename = Any[SSAValue(i) for i = 1:new_len]
         bb_rename = Vector{Int}()
         pending_nodes = NewNodeStream()
@@ -769,9 +758,7 @@ end
 
 function add_pending!(compact::IncrementalCompact, pos::Int, attach_after::Bool)
     node = add!(compact.pending_nodes, pos, attach_after)
-    # TODO: switch this to `l = length(pending_nodes); splice!(pending_perm, searchsorted(pending_perm, l), l)`
-    push!(compact.pending_perm, length(compact.pending_nodes))
-    sort!(compact.pending_perm, DEFAULT_STABLE, Order.By(x->compact.pending_nodes.info[x].pos, Order.Forward))
+    heappush!(compact.pending_perm, length(compact.pending_nodes), By(x -> compact.pending_nodes.info[x].pos))
     return node
 end
 
@@ -1456,7 +1443,7 @@ function iterate_compact(compact::IncrementalCompact, (idx, active_bb)::Tuple{In
             if !(info.attach_after ? info.pos <= compact.idx - 1 : info.pos <= compact.idx)
                 break
             end
-            popfirst!(compact.pending_perm)
+            heappop!(compact.pending_perm, By(x -> compact.pending_nodes.info[x].pos))
         end
         # Move to next block
         compact.idx += 1
@@ -1481,7 +1468,7 @@ function iterate_compact(compact::IncrementalCompact, (idx, active_bb)::Tuple{In
     elseif !isempty(compact.pending_perm) &&
         (info = compact.pending_nodes.info[compact.pending_perm[1]];
          info.attach_after ? info.pos == idx - 1 : info.pos == idx)
-        new_idx = popfirst!(compact.pending_perm)
+        new_idx = heappop!(compact.pending_perm, By(x -> compact.pending_nodes.info[x].pos))
         new_node_entry = compact.pending_nodes.stmts[new_idx]
         new_node_info = compact.pending_nodes.info[new_idx]
         new_idx += length(compact.ir.stmts) + length(compact.ir.new_nodes)

base/compiler/ssair/passes.jl

@@ -72,7 +72,7 @@ function try_compute_fieldidx_stmt(ir::Union{IncrementalCompact,IRCode}, stmt::E
     return try_compute_fieldidx(typ, field)
 end
 
-function find_curblock(domtree::DomTree, allblocks::Vector{Int}, curblock::Int)
+function find_curblock(domtree::DomTree, allblocks::BitSet, curblock::Int)
     # TODO: This can be much faster by looking at current level and only
     # searching for those blocks in a sorted order
     while !(curblock in allblocks) && curblock !== 0
@@ -92,7 +92,7 @@ function val_for_def_expr(ir::IRCode, def::Int, fidx::Int)
     end
 end
 
-function compute_value_for_block(ir::IRCode, domtree::DomTree, allblocks::Vector{Int}, du::SSADefUse, phinodes::IdDict{Int, SSAValue}, fidx::Int, curblock::Int)
+function compute_value_for_block(ir::IRCode, domtree::DomTree, allblocks::BitSet, du::SSADefUse, phinodes::IdDict{Int, SSAValue}, fidx::Int, curblock::Int)
     curblock = find_curblock(domtree, allblocks, curblock)
     def = 0
     for stmt in du.defs
@@ -103,7 +103,7 @@ function compute_value_for_block(ir::IRCode, domtree::DomTree, allblocks::Vector
     def == 0 ? phinodes[curblock] : val_for_def_expr(ir, def, fidx)
 end
 
-function compute_value_for_use(ir::IRCode, domtree::DomTree, allblocks::Vector{Int},
+function compute_value_for_use(ir::IRCode, domtree::DomTree, allblocks::BitSet,
     du::SSADefUse, phinodes::IdDict{Int, SSAValue}, fidx::Int, use::Int)
     def, useblock, curblock = find_def_for_use(ir, domtree, allblocks, du, use)
     if def == 0
@@ -122,7 +122,7 @@ end
 # even when the allocation contains an uninitialized field, we try an extra effort to check
 # if this load at `idx` have any "safe" `setfield!` calls that define the field
 function has_safe_def(
-    ir::IRCode, domtree::DomTree, allblocks::Vector{Int}, du::SSADefUse,
+    ir::IRCode, domtree::DomTree, allblocks::BitSet, du::SSADefUse,
     newidx::Int, idx::Int)
     def, _, _ = find_def_for_use(ir, domtree, allblocks, du, idx)
     # will throw since we already checked this `:new` site doesn't define this field
@@ -157,7 +157,7 @@ end
 
 # find the first dominating def for the given use
 function find_def_for_use(
-    ir::IRCode, domtree::DomTree, allblocks::Vector{Int}, du::SSADefUse, use::Int, inclusive::Bool=false)
+    ir::IRCode, domtree::DomTree, allblocks::BitSet, du::SSADefUse, use::Int, inclusive::Bool=false)
     useblock = block_for_inst(ir.cfg, use)
     curblock = find_curblock(domtree, allblocks, useblock)
     local def = 0
@@ -1306,7 +1306,7 @@ function sroa_mutables!(ir::IRCode, defuses::IdDict{Int, Tuple{SPCSet, SSADefUse
         # but we should come up with semantics for well defined semantics
         # for uninitialized fields first.
         ndefuse = length(fielddefuse)
-        blocks = Vector{Tuple{#=phiblocks=# Vector{Int}, #=allblocks=# Vector{Int}}}(undef, ndefuse)
+        blocks = Vector{Tuple{#=phiblocks=# Vector{Int}, #=allblocks=# BitSet}}(undef, ndefuse)
         for fidx in 1:ndefuse
             du = fielddefuse[fidx]
             isempty(du.uses) && continue
@@ -1317,7 +1317,7 @@ function sroa_mutables!(ir::IRCode, defuses::IdDict{Int, Tuple{SPCSet, SSADefUse
             else
                 phiblocks = iterated_dominance_frontier(ir.cfg, ldu, get!(lazydomtree))
             end
-            allblocks = sort!(vcat(phiblocks, ldu.def_bbs); alg=QuickSort)
+            allblocks = union!(BitSet(phiblocks), ldu.def_bbs)
             blocks[fidx] = phiblocks, allblocks
             if fidx + 1 > length(defexpr.args)
                 for i = 1:length(du.uses)

base/cpuid.jl

@@ -95,7 +95,7 @@ let
     arch = normalize_arch(String(Sys.ARCH))
     if arch in keys(ISAs_by_family)
         for isa in ISAs_by_family[arch]
-            unique!(append!(FEATURES, last(isa).features))
+            Base._unique!(append!(FEATURES, last(isa).features))
         end
     end
 

base/range.jl

@@ -1375,19 +1375,6 @@ function _reverse(r::StepRangeLen, ::Colon)
 end
 _reverse(r::LinRange{T}, ::Colon) where {T} = typeof(r)(r.stop, r.start, length(r))
 
-## sorting ##
-
-issorted(r::AbstractUnitRange) = true
-issorted(r::AbstractRange) = length(r) <= 1 || step(r) >= zero(step(r))
-
-sort(r::AbstractUnitRange) = r
-sort!(r::AbstractUnitRange) = r
-
-sort(r::AbstractRange) = issorted(r) ? r : reverse(r)
-
-sortperm(r::AbstractUnitRange) = 1:length(r)
-sortperm(r::AbstractRange) = issorted(r) ? (1:1:length(r)) : (length(r):-1:1)
-
 function sum(r::AbstractRange{<:Real})
     l = length(r)
     # note that a little care is required to avoid overflow in l*(l-1)/2

base/reflection.jl

@@ -100,7 +100,9 @@ since it is not idiomatic to explicitly export names from `Main`.
 See also: [`@locals`](@ref Base.@locals), [`@__MODULE__`](@ref).
 """
 names(m::Module; all::Bool = false, imported::Bool = false) =
-    sort!(ccall(:jl_module_names, Array{Symbol,1}, (Any, Cint, Cint), m, all, imported))
+    sort!(unsorted_names(m; all, imported))
+unsorted_names(m::Module; all::Bool = false, imported::Bool = false) =
+    ccall(:jl_module_names, Array{Symbol,1}, (Any, Cint, Cint), m, all, imported)
 
 isexported(m::Module, s::Symbol) = ccall(:jl_module_exports_p, Cint, (Any, Any), m, s) != 0
 isdeprecated(m::Module, s::Symbol) = ccall(:jl_is_binding_deprecated, Cint, (Any, Any), m, s) != 0

base/sysimg.jl

@@ -49,14 +49,15 @@ let
         :Distributed,
         :Future,
         :InteractiveUtils,
-        :LibGit2,
         :Profile,
-        :SparseArrays,
+        :Sort,
         :UUIDs,
 
         # 3-depth packages
+        :LibGit2,
         :REPL,
         :SharedArrays,
+        :SparseArrays,
         :TOML,
         :Test,
 

base/util.jl

@@ -48,10 +48,9 @@ const disable_text_style = Dict{Symbol,String}(
 # of colors.
 let color_syms = collect(Iterators.filter(x -> !isa(x, Integer), keys(text_colors))),
     formatting_syms = [:normal, :bold, :default]
-    global const available_text_colors = cat(
-        sort!(intersect(color_syms, formatting_syms), rev=true),
-        sort!(setdiff(  color_syms, formatting_syms));
-        dims=1)
+    global const available_text_colors = vcat(
+        Core.Compiler.sort!(intersect(color_syms, formatting_syms); < = >), #reverse
+        Core.Compiler.sort!(  setdiff(color_syms, formatting_syms); <))
 end
 
 const available_text_colors_docstring =

contrib/print_sorted_stdlibs.jl

@@ -27,8 +27,8 @@ end
 
 project_deps = Dict{String,Set{String}}()
 for project_dir in readdir(STDLIB_DIR, join=true)
-    files = readdir(project_dir)
-    if "Project.toml" in files
+    project_file = joinpath(project_dir, "Project.toml")
+    if isfile(project_file)
         project = TOML.parsefile(joinpath(project_dir, "Project.toml"))
 
         if !haskey(project, "name")

doc/make.jl

@@ -116,7 +116,6 @@ BaseDocs = [
     "base/file.md",
     "base/io-network.md",
     "base/punctuation.md",
-    "base/sort.md",
     "base/iterators.md",
     "base/c.md",
     "base/libc.md",