rafaqz · rafaqz · Mar 8, 2025 · Mar 8, 2025 · Mar 8, 2025 · Mar 26, 2025
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,77 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Links are updated with Changelog.jl using the command:
+
+```julia
+Changelog.generate(
+    Changelog.CommonMark(),             # output type
+    "CHANGELOG.md";                     # input and output file
+    repo = "rafaqz/DimensionalData.jl", # default repository for links
+)
+```
+
+## [0.30.0]
+
+### Added
+
+- `combine` method added to complement `groupby` ([#903])
+- `DimSlices` object is now documented interface.
+- Tables can now be converted to `AbstractDimArray` or `AbstractDimStack`,
+  guessing the lookup values from dimension columns, thanks to major efforts by [@JoshuaBillson] and [@tiemvanderdeure] in ([#739])
+- `presevedims` can be used to preseve dimensions of array during conversion to
+  tables, so that values may themselves be `AbstractDimArray` ([#917])
+- `rebuildsliced` documented and added to the developer interface 
+- Broadcasts improved for all `AbstractBasicDimArray`, like `DimSelectors`.
+- `groupby` accepts name keyword to set the name of the DimGroupbyArray
+
+
+### Fixed
+
+- `Begin`/`End` bugfixed for use as types
+- lookup methods like `order`, `span` etc work from all objects consistently
+  ([#1071])
+
+### Changed
+
+- `similar` for `AbstractDimStack` has modified behavior more consistent with
+  `similar` for `AbstractArray` ([#903]).
+- Internal const `CategoricalEltypes` now includes `DataType` - so that a
+  lookup of types is considered `Categorical` by default ([#876]).
+- `skipmissing` on an `AbstractDimStack` now skips any `missing` values in any fiels, rather than `missing`, which can't
+  actually occur ([#1041]).
+- Minor changes in coversion to and from tables in some cases.
+- Abstract constructors for AbstractDimArray were removed, as they cause ambiguity hassles. 
+  If necessary, extending packages can define methods like these for their own types:
+    DimArray{T}(x::UndefInitializer, dims::Dimension...; kw...) where T = DimArray{T}(x, dims; kw...)
+    DimArray{T}(x::UndefInitializer, dims::MaybeDimTuple; kw...) where T = DimArray(Array{T}(undef, map(length, dims)), dims; kw...)
+    MyDimArray(st::AbstractDimStack; kw...) = DD.dimarray_from_dimstack(MyDimArray, st; kw...) 
+- the deprecated `index` function is now removed completely
+- `metadata(obj, dims)` no longer works as its ambiguous and not consistently
+  implemented - `metadata(obj)` does not return a Tuple for each dimension 
+  like other similar methods, but the metadata of `obj`. To get dimension 
+  metadata going forward, explicitly use `metadata(dims(obj, X))`
+- `val(obj)` no longer returns `map(val, dims(obj))` because that's just weird,
+  val seems like it would return `parent` from that call. `lookup` does this anyway.
+- Added a `D` parameter to AbstractDimStack in ([#1128]) so that its more
+  interoperable with AbstractDimArray
+- Removed two methods that don't have much use and cause lots of invalidations:
+  - `Base.convert(Type{<:AbstractString}, name::DD.AbstractName)` is removed, use `string(name)` instead.
+  - `Base.merge(stack::AbstractDimStack, pairs)` is removed, use `merge(stack, NamedTuple(pairs))` instead.
+
+
+<!-- Links generated by Changelog.jl -->
+
+[#739]: https://github.com/rafaqz/DimensionalData.jl/issues/739
+[#876]: https://github.com/rafaqz/DimensionalData.jl/issues/876
+[#903]: https://github.com/rafaqz/DimensionalData.jl/issues/903
+[#917]: https://github.com/rafaqz/DimensionalData.jl/issues/917
+[#1041]: https://github.com/rafaqz/DimensionalData.jl/issues/1041
+[#1071]: https://github.com/rafaqz/DimensionalData.jl/issues/1071
+[#1128]: https://github.com/rafaqz/DimensionalData.jl/issues/1128
+[@JoshuaBillson]: https://github.com/JoshuaBillson
+[@tiemvanderdeure]: https://github.com/tiemvanderdeure
diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "DimensionalData"
 uuid = "0703355e-b756-11e9-17c0-8b28908087d0"
 authors = ["Rafael Schouten <[email protected]>"]
-version = "0.29.24"
+version = "0.30.0"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"

diff --git a/docs/src/api/reference.md b/docs/src/api/reference.md
@@ -59,6 +59,7 @@ prune
 DimIndices
 DimSelectors
 DimPoints
+DimensionalData.DimSlices
 ```
 
 ## Tables.jl/TableTraits.jl interface
@@ -68,12 +69,13 @@ DimensionalData.AbstractDimTable
 DimTable
 ```
 
-# Group by methods
+## Group by methods
 
 For transforming DimensionalData objects:
 
 ```@docs
 groupby
+combine
 DimensionalData.DimGroupByArray
 Bins
 ranges
@@ -84,7 +86,7 @@ months
 hours
 ```
 
-# Utility methods
+## Utility methods
 
 For transforming DimensionalData objects:
 
@@ -100,7 +102,7 @@ unmergedims
 reorder
 ```
 
-# Global lookup strictness settings
+## Global lookup strictness settings
 
 Control how strict DimensionalData when comparing [`Lookup`](@ref)s
 before doing broadcasts and matrix multipications.
@@ -117,12 +119,13 @@ DimensionalData.strict_matmul
 DimensionalData.strict_matmul!
 ```
 
-Base methods
+## Base methods
 
 ```@docs
 Base.cat
 Base.copy!
 Base.eachslice
+Base.skipmissing
 ```
 
 Most base methods work as expected, using `Dimension` wherever a `dims`
@@ -142,6 +145,7 @@ DimensionalData.NoName
 DimensionalData.DimArrayInterface
 DimensionalData.DimStackInterface
 DimensionalData.rebuild_from_arrays
+DimensionalData.rebuildsliced
 DimensionalData.show_main
 DimensionalData.show_after
 DimensionalData.refdims_title

diff --git a/docs/src/tables.md b/docs/src/tables.md
@@ -6,12 +6,22 @@ Description = "Convert DimensionalData.jl arrays to Tables.jl format and DataFra
 
 [Tables.jl](https://github.com/JuliaData/Tables.jl) provides an ecosystem-wide interface to tabular data in Julia, ensuring interoperability with [DataFrames.jl](https://dataframes.juliadata.org/stable/), [CSV.jl](https://csv.juliadata.org/stable/), and hundreds of other packages that implement the standard.
 
+## Dimensional data are tables
 DimensionalData.jl implements the Tables.jl interface for `AbstractDimArray` and `AbstractDimStack`. `DimStack` layers are unrolled so they are all the same size, and dimensions loop to match the length of the largest layer.
 
 Columns are given the [`name`](@ref) of the array or stack layer, and the result of `DD.name(dimension)` for `Dimension` columns.
 
-Looping of dimensions and stack layers is done _lazily_,
-and does not allocate unless collected.
+Looping of dimensions and stack layers is done _lazily_, and does not allocate unless collected.
+
+## Materializing tables to DimArray or DimStack
+`DimArray` and `DimStack` have fallback methods to materialize any `Tables.jl`-compatible table.
+
+By default, it will treat columns such as X, Y, Z, and Band as dimensions, and other columns as data.
+Pass a `name` keyword argument to determine which column(s) are used.
+
+You have full control over which columns are dimensions - and what those dimensions look like exactly. If you pass a `Tuple` of `Symbol` or dimension types (e.g. `X`) as the second argument, those columns are treated as dimensions. Passing a `Tuple` of dimensions preserves these dimensions - with values matched to the corresponding columns.
+
+Materializing tables will worked even if the table is not ordered, and can handle missing values.
 
 ## Example
 
@@ -93,3 +103,61 @@ using CSV
 CSV.write("dimstack.csv", st)
 readlines("dimstack.csv")
 ````
+
+## Converting a DataFrame to a DimArray or DimStack
+
+The Dataframe we use will have 5 columns: X, Y, category, data1, and data2
+
+````@ansi dataframe
+df = DataFrame(st)
+````
+
+::: tabs
+
+== Create a `DimArray`
+
+Converting this DataFrame to a DimArray without other arguments will read the `category` columns as data and ignore data1 and data2:
+
+````@ansi dataframe
+DimArray(df)
+````
+
+Specify dimenion names to ensure these get treated as dimensions. Now data1 is read in instead.
+````@ansi dataframe
+DimArray(df, (X,Y,:category))
+````
+
+You can also pass in the actual dimensions.
+````@ansi dataframe
+DimArray(df, dims(st))
+````
+
+Pass in a name argument to read in data2 instead.
+````@ansi dataframe
+DimArray(df, dims(st); name = :data2)
+````
+
+== Create a `DimStack`
+
+Converting the DataFrame to a `DimStack` will by default read category, data1, and data2 as layers
+````@ansi dataframe
+DimStack(df)
+````
+
+
+Specify dimenion names to ensure these get treated as dimensions. Now data1 and data2 are layers.
+````@ansi dataframe
+DimStack(df, (X,Y,:category))
+````
+
+You can also pass in the actual dimensions. 
+````@ansi dataframe
+DimStack(df, dims(st))
+````
+
+Pass in a tuple of column names to control which columns are read.
+````@ansi dataframe
+DimStack(df, dims(st); name = (:data2,))
+````
+
+:::
diff --git a/ext/DimensionalDataAlgebraOfGraphicsExt.jl b/ext/DimensionalDataAlgebraOfGraphicsExt.jl
@@ -30,18 +30,19 @@ end
 # so that e.g. `X` also applies to any `XDim`, and so forth.
 function AoG.select(data::AoG.Columns{<: DD.AbstractDimTable}, dim::DimOrType)
     # Query the dimensions in the table for the dimension
-    available_dimension = DD.dims(data.columns, dim)
-    # If the dimension is not found, it might be the name of the 
+    alldims = DD._dims(data.columns)
+    available_dimension = DD.dims(alldims, dim)
+    # If the dimension is not found, it might be the name of the
     # underlying array.
     name = if isnothing(available_dimension)
         if DD.name(dim) in DD.name(parent(data.columns))
             error(
-                "Dimension $dim not found in DimTable with dimensions $(DD.dims(data.columns)), **but** it is" *
+                "Dimension $dim not found in DimTable with dimensions $(alldims), **but** it is" *
                 (length(DD.name(parent(data.columns))) > 1 ? "a layer of the parent array" : "the name of the parent array") *
                 ".\n\nYou can pass this directly as a `Symbol`, i.e. **`:$(DD.name(dim))`**."
             )
         else
-            error("Dimension $dim not found in DimTable with dimensions $(DD.dims(data.columns)), and neither was it the name of the array ($(DD.name(parent(data.columns)))).")
+            error("Dimension $dim not found in DimTable with dimensions $(alldims), and neither was it the name of the array ($(DD.name(parent(data.columns)))).")
         end
     else
         # The dimension was found, so use that name.

diff --git a/ext/DimensionalDataMakieExt.jl b/ext/DimensionalDataMakieExt.jl
@@ -3,7 +3,7 @@ module DimensionalDataMakieExt
 using DimensionalData
 using Makie
 using IntervalSets
-using DimensionalData.Dimensions, DimensionalData.LookupArrays
+using DimensionalData.Dimensions, DimensionalData.Lookups
 
 const DD = DimensionalData
 

diff --git a/src/DimensionalData.jl b/src/DimensionalData.jl
@@ -36,8 +36,9 @@ include("Dimensions/Dimensions.jl")
 using .Dimensions
 using .Dimensions.Lookups
 using .Dimensions: StandardIndices, DimOrDimType, DimTuple, DimTupleOrEmpty, DimType, AllDims
+using .Dimensions: INTERFACE_QUERY_FUNCTION_NAMES
 import .Lookups: metadata, set, _set, rebuild, basetypeof, 
-    order, span, sampling, locus, val, index, bounds, intervalbounds,
+    order, span, sampling, locus, val, bounds, intervalbounds,
     hasselection, units, SelectorOrInterval, Begin, End
 import .Dimensions: dims, refdims, name, lookup, kw2dims, hasdim, label, _astuple
 
@@ -47,11 +48,6 @@ import DataAPI.groupby
 
 export Lookups, Dimensions
 
-# Deprecated
-const LookupArrays = Lookups
-const LookupArray = Lookup
-export LookupArrays, LookupArray
-
 # Dimension
 export X, Y, Z, Ti, Dim, Coord
 
@@ -70,8 +66,7 @@ export AbstractDimTable, DimTable
 
 export AbstractDimTree, DimTree, prune
 
-export DimIndices, DimSelectors, DimPoints, #= deprecated =# DimKeys
-
+export DimIndices, DimSelectors, DimPoints
 # getter methods
 export dims, refdims, metadata, name, lookup, bounds, val, layers
 
@@ -82,7 +77,7 @@ export dimnum, hasdim, hasselection, otherdims
 export set, rebuild, reorder, modify, broadcast_dims, broadcast_dims!,
     mergedims, unmergedims, maplayers
 
-export groupby, seasons, months, hours, intervals, ranges
+export groupby, combine, seasons, months, hours, intervals, ranges
 
 
 export @d
@@ -92,6 +87,7 @@ const DD = DimensionalData
 # Common
 include("interface.jl")
 include("name.jl")
+include("table_ops.jl")
 
 # Arrays
 include("array/array.jl")
@@ -115,6 +111,7 @@ include("tables.jl")
 include("plotrecipes.jl")
 include("utils.jl")
 include("set.jl")
+include("opaque.jl")
 include("groupby.jl")
 include("precompile.jl")
 include("interface_tests.jl")

diff --git a/src/Dimensions/Dimensions.jl b/src/Dimensions/Dimensions.jl
@@ -20,16 +20,13 @@ include("../Lookups/Lookups.jl")
 
 using .Lookups
 
-const LU = Lookups
-const LookupArrays = Lookups
-
-import .Lookups: rebuild, order, span, sampling, locus, val, index, set, _set,
+import .Lookups: rebuild, order, span, sampling, locus, val, set, _set,
     metadata, bounds, intervalbounds, units, basetypeof, unwrap, selectindices, hasselection,
     shiftlocus, maybeshiftlocus, ordered_first, ordered_last, ordered_firstindex, ordered_lastindex, 
     promote_first, _remove
 using .Lookups: StandardIndices, SelTuple, CategoricalEltypes,
     LookupTrait, AllMetadata, LookupSetters, AbstractBeginEndRange,
-    SelectorOrInterval, Interval
+    SelectorOrInterval, Interval, Begin, End, LazyMath 
 
 using Base: tail, OneTo, @propagate_inbounds