Breaking

docs/src/api/reference.md

@@ -55,6 +55,7 @@ prune
 DimIndices
 DimSelectors
 DimPoints
+DimSlices
 ```
 
 ## Tables.jl/TableTraits.jl interface
@@ -70,6 +71,7 @@ For transforming DimensionalData objects:
 
 ```@docs
 groupby
+combine
 DimensionalData.DimGroupByArray
 Bins
 ranges
@@ -138,6 +140,7 @@ DimensionalData.NoName
 DimensionalData.DimArrayInterface
 DimensionalData.DimStackInterface
 DimensionalData.rebuild_from_arrays
+DimensionalData.rebuildsliced
 DimensionalData.show_main
 DimensionalData.show_after
 DimensionalData.refdims_title

src/DimensionalData.jl

@@ -71,8 +71,7 @@ export AbstractDimTable, DimTable
 
 export AbstractDimTree, DimTree, prune
 
-export DimIndices, DimSelectors, DimPoints, #= deprecated =# DimKeys
-
+export DimIndices, DimSelectors, DimPoints
 # getter methods
 export dims, refdims, metadata, name, lookup, bounds, val, layers
 
@@ -83,7 +82,7 @@ export dimnum, hasdim, hasselection, otherdims
 export set, rebuild, reorder, modify, broadcast_dims, broadcast_dims!,
     mergedims, unmergedims, maplayers
 
-export groupby, seasons, months, hours, intervals, ranges
+export groupby, combine, seasons, months, hours, intervals, ranges
 
 
 export @d
@@ -116,6 +115,7 @@ include("tables.jl")
 include("plotrecipes.jl")
 include("utils.jl")
 include("set.jl")
+include("opaque.jl")
 include("groupby.jl")
 include("precompile.jl")
 include("interface_tests.jl")

src/Dimensions/dimension.jl

@@ -273,6 +273,7 @@
 Base.eachindex(d::Dimension) = eachindex(val(d))
 Base.length(d::Dimension) = length(val(d))
 Base.ndims(d::Dimension) = 0
+Base.parentindices(d::Dimension{<:AbstractArray}) = parentindices(parent(d))
 Base.ndims(d::Dimension{<:AbstractArray}) = ndims(val(d))
 Base.iterate(d::Dimension{<:AbstractArray}, args...) = iterate(lookup(d), args...)
 Base.first(d::Dimension) = val(d)

src/Lookups/Lookups.jl

@@ -59,17 +59,15 @@ export Unaligned, Transformed, ArrayLookup
 # Deprecated
 export LookupArray
 
-const StandardIndices = Union{AbstractArray{<:Integer},Colon,Integer,CartesianIndex,CartesianIndices}
-
 # As much as possible keyword rebuild is automatic
 rebuild(x; kw...) = ConstructionBase.setproperties(x, (; kw...))
 
-include("metadata.jl")
 include("lookup_traits.jl")
+include("metadata.jl")
 include("lookup_arrays.jl")
+include("beginend.jl")
 include("predicates.jl")
 include("selector.jl")
-include("beginend.jl")
 include("indexing.jl")
 include("methods.jl")
 include("utils.jl")

src/Lookups/beginend.jl

@@ -45,6 +45,8 @@ Base.to_indices(A, inds, (r, args...)::Tuple{<:Union{Begin,End,<:LazyMath},Varar
 _to_index(inds, a::Int) = a
 _to_index(inds, ::Begin) = first(inds)
 _to_index(inds, ::End) = last(inds)
+_to_index(inds, ::Type{Begin}) = first(inds)
+_to_index(inds, ::Type{End}) = last(inds)
 _to_index(inds, l::LazyMath{End}) = l.f(last(inds))
 _to_index(inds, l::LazyMath{Begin}) = l.f(first(inds))
 

src/Lookups/lookup_arrays.jl

@@ -32,6 +32,7 @@
 Base.last(l::Lookup) = last(parent(l))
 Base.firstindex(l::Lookup) = firstindex(parent(l))
 Base.lastindex(l::Lookup) = lastindex(parent(l))
+Base.parentindices(l::Lookup) = parentindices(parent(l))
 function Base.:(==)(l1::Lookup, l2::Lookup)
     basetypeof(l1) == basetypeof(l2) && parent(l1) == parent(l2)
 end
@@ -159,11 +160,10 @@
 rebuild(l::NoLookup; data=parent(l), kw...) = NoLookup(data)
 
 # Used in @d broadcasts
-struct Length1NoLookup <: AbstractNoLookup end
-Length1NoLookup(::AbstractVector) = Length1NoLookup()
-
-rebuild(l::Length1NoLookup; kw...) = Length1NoLookup()
-Base.parent(::Length1NoLookup) = Base.OneTo(1)
+struct Length1NoLookup{A<:AbstractUnitRange} <: AbstractNoLookup 
+    data::A
+end
+Length1NoLookup() = Length1NoLookup(Base.OneTo(1))
 
 """
     AbstractSampled <: Aligned
@@ -468,7 +468,7 @@
 order(lookup::AbstractCategorical) = lookup.order
 metadata(lookup::AbstractCategorical) = lookup.metadata
 
-const CategoricalEltypes = Union{AbstractChar,Symbol,AbstractString}
+const CategoricalEltypes = Union{AbstractChar,Symbol,AbstractString,DataType}
 
 function Adapt.adapt_structure(to, l::AbstractCategorical)
     rebuild(l; data=Adapt.adapt(to, parent(l)), metadata=NoMetadata())
@@ -866,6 +866,7 @@
 # Fallback NoLookup if not identical type
 promote_first(l1::Lookup) = l1
 promote_first(l1::L, ls::L...) where L<:Lookup = rebuild(l1; metadata=NoMetadata)
+promote_first(l1::L, ls::L...) where L<:AbstractNoLookup = l1
 function promote_first(l1::Lookup, ls1::Lookup...)
     ls = _remove(Length1NoLookup, l1, ls1...)
     if length(ls) != length(ls1) + 1

src/Lookups/selector.jl

@@ -1,3 +1,12 @@
+const StandardIndices = Union{
+    AbstractArray{<:Integer},
+    Colon,
+    Integer,
+    CartesianIndex,
+    CartesianIndices,
+    BeginEndRange,
+}
+
 struct SelectorError{L,S} <: Exception
     lookup::L
     selector::S

src/Lookups/utils.jl

@@ -111,8 +111,6 @@ end
 _order(A) = first(A) <= last(A) ? ForwardOrdered() : ReverseOrdered()
 _order(A::AbstractArray{<:IntervalSets.Interval}) = first(A).left <= last(A).left ? ForwardOrdered() : ReverseOrdered()
 
-@deprecate maybeshiftlocus maybeshiftlocus
-@deprecate shiftlocus shiftlocus
 
 # Remove objects of type T from a 
 Base.@assume_effects :foldable _remove(::Type{T}, x, xs...) where T = (x, _remove(T, xs...)...)

src/array/array.jl

@@ -1,4 +1,5 @@
 const IDim = Dimension{<:StandardIndices}
+const MaybeDimTuple = Tuple{Vararg{Dimension}}
 
 """
     AbstractBasicDimArray <: AbstractArray
@@ -8,7 +9,7 @@
 
 Only keyword `rebuild` is guaranteed to work with `AbstractBasicDimArray`.
 """
-abstract type AbstractBasicDimArray{T,N,D<:Tuple{Vararg{Dimension}}} <: AbstractArray{T,N} end
+abstract type AbstractBasicDimArray{T,N,D<:MaybeDimTuple} <: AbstractArray{T,N} end
 
 const AbstractBasicDimVector = AbstractBasicDimArray{T,1} where T
 const AbstractBasicDimMatrix = AbstractBasicDimArray{T,2} where T
@@ -94,10 +95,27 @@
 
 layerdims(A::AbstractDimArray) = basedims(A)
 
-@inline rebuildsliced(A::AbstractBasicDimArray, args...) = rebuildsliced(getindex, A, args...)
-@inline function rebuildsliced(f::Function, A::AbstractBasicDimArray, data::AbstractArray, I::Tuple, name=name(A))
+"""
+    rebuildsliced(f::Function, A::AbstractBasicDimArray, I)
+
+Rebuild `AbstractDimArray` where `f` is `getindex` , `view` or `dotview`.
+
+This does not need to be defined for `AbstractDimArray`, as `f`
+is simply called on the parent array, dims and refdims are sliced with `slicedims`, 
+and `rebuild` is called.
+
+However for custom `AbstractBasicDimArray`, `rebuildsliced` methods are needed
+to define slicing behavior, as there not be a parent array.
+"""
+@propagate_inbounds rebuildsliced(A::AbstractBasicDimArray, args...) = rebuildsliced(getindex, A, args...)
+@propagate_inbounds function rebuildsliced(f::Function, A::AbstractDimArray, I::Tuple, name=name(A))
+    I1 = to_indices(A, I)
+    data = f(parent(A), I1...)
+    return rebuildsliced(f, A, data, I1, name)
+end
+@propagate_inbounds function rebuildsliced(f::Function, A::AbstractDimArray, data::AbstractArray, I::Tuple, name=name(A))
     I1 = to_indices(A, I)
-    rebuild(A, data, slicedims(f, A, I1)..., name)
+    return rebuild(A, data, slicedims(f, A, I1)..., name)
 end
 
 # Array interface methods ######################################################
@@ -107,6 +125,7 @@
 Base.iterate(A::AbstractDimArray, args...) = iterate(parent(A), args...)
 Base.IndexStyle(A::AbstractDimArray) = Base.IndexStyle(parent(A))
 Base.parent(A::AbstractDimArray) = data(A)
+Base.parentindices(A::AbstractDimArray) = parentindices(parent(A))
 Base.vec(A::AbstractDimArray) = vec(parent(A))
 # Only compare data and dim - metadata and refdims can be different
 Base.:(==)(A1::AbstractDimArray, A2::AbstractDimArray) =
@@ -170,14 +189,6 @@
 # An alternative would be to fill missing dims with `Anon`, and keep existing
 # dims but strip the Lookup? It just seems a little complicated when the methods
 # below using DimTuple work better anyway.
-Base.similar(A::AbstractDimArray, i::Integer, I::Vararg{Integer}; kw...) =
-    similar(A, eltype(A), (i, I...); kw...)
-Base.similar(A::AbstractDimArray, I::Tuple{Int,Vararg{Int}}; kw...) = 
-    similar(A, eltype(A), I; kw...)
-Base.similar(A::AbstractDimArray, ::Type{T}, i::Integer, I::Vararg{Integer}; kw...) where T =
-    similar(A, T, (i, I...); kw...)
-Base.similar(A::AbstractDimArray, ::Type{T}, I::Tuple{Int,Vararg{Int}}; kw...) where T =
-    similar(parent(A), T, I)
 
 const MaybeDimUnitRange = Union{Integer,Base.OneTo,Dimensions.DimUnitRange}
 # when all axes are DimUnitRanges we can return an `AbstractDimArray`
@@ -256,14 +267,27 @@
 end
 
 # With Dimensions we can return an `AbstractDimArray`
-Base.similar(A::AbstractBasicDimArray, D::DimTuple; kw...) = Base.similar(A, eltype(A), D; kw...) 
-Base.similar(A::AbstractBasicDimArray, D::Dimension...; kw...) = Base.similar(A, eltype(A), D; kw...) 
-Base.similar(A::AbstractBasicDimArray, ::Type{T}, D::Dimension...; kw...) where T =
-    Base.similar(A, T, D; kw...) 
+Base.similar(A::AbstractBasicDimArray, d1::Dimension, D::Dimension...; kw...) =
+    Base.similar(A, eltype(A), (d1, D...); kw...)
+Base.similar(A::AbstractBasicDimArray, ::Type{T}, d1::Dimension, D::Dimension...; kw...) where T =
+    Base.similar(A, T, (d1, D...); kw...)
+Base.similar(A::AbstractBasicDimArray, D::DimTuple; kw...) = 
+    Base.similar(A, eltype(A), D; kw...) 
+function Base.similar(A::AbstractBasicDimArray, ::Type{T}, D::DimTuple; kw...) where T
+    data = _arraytype(T)(undef, _dimlength(D))
+    dimconstructor(D)(data, D; kw...)
+end
+function Base.similar(A::AbstractBasicDimArray, ::Type{T}, D::Tuple{};
+    refdims=(), name=_noname(A), metadata=NoMetadata(), kw...
+) where T
+    data = _arraytype(T)(undef, _dimlength(D))
+    dimconstructor(D)(data, (); refdims, name, metadata, kw...)
+end
+
 function Base.similar(A::AbstractDimArray, ::Type{T}, D::DimTuple; 
     refdims=(), name=_noname(A), metadata=NoMetadata(), kw...
 ) where T
-    data = similar(parent(A), T, _dimlength(D))
+    data = _arraytype(T)(undef, _dimlength(D))
     dims = _maybestripval(D)
     return rebuild(A; data, dims, refdims, metadata, name, kw...)
 end
@@ -274,6 +298,20 @@
     rebuild(A; data, dims=(), refdims, metadata, name, kw...)
 end
 
+Base.similar(A::AbstractBasicDimArray, shape::Int...; kw...) =
+    similar(A, eltype(A), shape; kw...)
+Base.similar(A::AbstractBasicDimArray, shape::Tuple{Vararg{Int}}; kw...) = 
+    similar(A, eltype(A), shape; kw...)
+Base.similar(A::AbstractBasicDimArray, ::Type{T}, shape::Int...; kw...) where T =
+    similar(A, T, shape; kw...)
+Base.similar(A::AbstractBasicDimArray, ::Type{T}, shape::Tuple{Vararg{Int}}; kw...) where T =
+    _arraytype(T)(undef, shape)
+Base.similar(A::AbstractDimArray, ::Type{T}, shape::Tuple{Vararg{Int}}; kw...) where T =
+    similar(parent(A), T, shape)
+
+_arraytype(::Type{T}) where T = Array{T}
+_arraytype(::Type{Bool}) = BitArray
+
 # Keep the same type in `similar`
 _noname(A::AbstractBasicDimArray) = _noname(name(A))
 _noname(s::String) = ""

src/array/broadcast.jl

@@ -35,7 +35,9 @@
 # It preserves the dimension names.
 # `S` should be the `BroadcastStyle` of the wrapped type.
 # Copied from NamedDims.jl (thanks @oxinabox).
-struct DimensionalStyle{S <: BroadcastStyle} <: AbstractArrayStyle{Any} end
+struct BasicDimensionalStyle{N} <: AbstractArrayStyle{Any} end
+
+struct DimensionalStyle{S<:BroadcastStyle} <: AbstractArrayStyle{Any} end
 DimensionalStyle(::S) where {S} = DimensionalStyle{S}()
 DimensionalStyle(::S, ::Val{N}) where {S,N} = DimensionalStyle(S(Val(N)))
 DimensionalStyle(::Val{N}) where N = DimensionalStyle{DefaultArrayStyle{N}}()
@@ -53,6 +55,8 @@
     inner_style = typeof(BroadcastStyle(A))
     return DimensionalStyle{inner_style}()
 end
+BroadcastStyle(::Type{<:AbstractBasicDimArray{T,N}}) where {T,N} =
+    BasicDimensionalStyle{N}()
 
 BroadcastStyle(::DimensionalStyle, ::Base.Broadcast.Unknown) = Unknown()
 BroadcastStyle(::Base.Broadcast.Unknown, ::DimensionalStyle) = Unknown()
@@ -79,12 +83,31 @@
     dims = format(Dimensions.promotedims(bdims...; skip_length_one=true), data)
     return rebuild(A; data, dims, refdims=refdims(A), name=Symbol(""))
 end
+function Broadcast.copy(bc::Broadcasted{BasicDimensionalStyle{N}}) where N
+    A = _firstdimarray(bc)
+    data = collect(bc)
+    A isa Nothing && return data # No AbstractDimArray
+
+    bdims = _broadcasted_dims(bc)
+    _comparedims_broadcast(A, bdims...)
+
+    data isa AbstractArray || return data # result is a scalar
+
+    # Return an AbstractDimArray
+    dims = format(Dimensions.promotedims(bdims...; skip_length_one=true), data)
+    return dimconstructor(dims)(data, dims; refdims=refdims(A), name=Symbol(""))
+end
 
 function Base.copyto!(dest::AbstractArray, bc::Broadcasted{DimensionalStyle{S}}) where S
     fda = _firstdimarray(bc) 
     isnothing(fda) || _comparedims_broadcast(fda, _broadcasted_dims(bc)...)
     copyto!(dest, _unwrap_broadcasted(bc))
 end
+function Base.copyto!(dest::AbstractArray, bc::Broadcasted{BasicDimensionalStyle{N}}) where N
+    fda = _firstdimarray(bc) 
+    isnothing(fda) || _comparedims_broadcast(fda, _broadcasted_dims(bc)...)
+    copyto!(dest, bc)
+end
 
 @inline function Base.Broadcast.materialize!(dest::AbstractDimArray, bc::Base.Broadcast.Broadcasted{<:Any})
     # Need to check whether the dims are compatible in dest, 
@@ -97,7 +120,15 @@
 
 function Base.similar(bc::Broadcast.Broadcasted{DimensionalStyle{S}}, ::Type{T}) where {S,T}
     A = _firstdimarray(bc)
-    rebuildsliced(A, similar(_unwrap_broadcasted(bc), T, axes(bc)...), axes(bc), Symbol(""))
+    data = similar(_unwrap_broadcasted(bc), T, size(bc))
+    dims, refdims = slicedims(A, axes(bc))
+    return rebuild(A; data, dims, refdims, name=Symbol(""))
+end
+function Base.similar(bc::Broadcast.Broadcasted{BasicDimensionalStyle{N}}, ::Type{T}) where {N,T}
+    A = _firstdimarray(bc)
+    data = similar(A, T, size(bc))
+    dims, refdims = slicedims(A, axes(bc))
+    return dimconstructor(dims)(data, dims; refdims, name=Symbol(""))
 end
 
 
@@ -386,9 +417,10 @@
 
 # Get the first dimensional array in the broadcast
 _firstdimarray(x::Broadcasted) = _firstdimarray(x.args)
-_firstdimarray(x::Tuple{<:AbstractDimArray,Vararg}) = x[1]
+_firstdimarray(x::Tuple{<:AbstractBasicDimArray,Vararg}) = x[1]
+_firstdimarray(x::AbstractBasicDimArray) = x
 _firstdimarray(ext::Base.Broadcast.Extruded) = _firstdimarray(ext.x)
-function _firstdimarray(x::Tuple{<:Broadcasted,Vararg})
+function _firstdimarray(x::Tuple{<:Union{Broadcasted,Base.Broadcast.Extruded},Vararg})
     found = _firstdimarray(x[1])
     if found isa Nothing
         _firstdimarray(tail(x))