Merge pull request #10 from JuliaGeo/format

format and clean up code

Author: rafaqz

docs/make.jl

@@ -10,11 +10,7 @@ makedocs(;
         canonical="https://juliageo.github.io/ZarrDatasets.jl",
         assets=String[],
     ),
-    pages=[
-        "Home" => "index.md",
-    ],
+    pages=["Home" => "index.md"],
 )
 
-deploydocs(;
-    repo="github.com/JuliaGeo/ZarrDatasets.jl",
-)
+deploydocs(; repo="github.com/JuliaGeo/ZarrDatasets.jl")

docs/src/index.md

@@ -1,7 +1,5 @@
-
 ## ZarrDatasets
 
-
 See the [documentation of JuliaGeo/CommonDataModel.jl](https://juliageo.org/CommonDataModel.jl/stable/) for the full documentation of the API. As a quick reference, here is an example how to create and read a Zarr file store as a quick reference.
 
 ### Create a Zarr file store
@@ -39,15 +37,10 @@ data_units = ds["varname"].attrib["units"]
 ```
 
 
-
 ```@autodocs
 Modules = [ZarrDatasets]
 ```
 
-
-
-
-
 ### Differences between Zarr and NetCDF files
 
 * All metadata (in particular attributes) is stored in JSON files for the Zarr format with the following implications:

src/ZarrDatasets.jl

@@ -1,10 +1,6 @@
 module ZarrDatasets
 
-import Base:
-    checkbounds,
-    getindex,
-    setindex!,
-    size
+import Base: checkbounds, getindex, setindex!, size
 
 import CommonDataModel:
     CFVariable,
@@ -26,18 +22,19 @@ import CommonDataModel:
     parentdataset,
     variable
 
-import DiskArrays:
-    eachchunk,
-    haschunks
+import DiskArrays: eachchunk, haschunks
 
 import CommonDataModel as CDM
+using JSON: JSON
+
 using DataStructures
 using Zarr
-import JSON
+
+export ZarrDataset
+export defDim, defVar, defGroup
 
 include("types.jl")
 include("dataset.jl")
 include("variable.jl")
 
-export ZarrDataset
 end

src/dataset.jl

@@ -1,14 +1,22 @@
+# Base interface
 
-CDM.name(v::ZarrDataset) = Zarr.zname(v.zgroup)
 Base.keys(ds::ZarrDataset) = keys(ds.zgroup.arrays)
-Base.haskey(ds::ZarrDataset,varname::SymbolOrString) = haskey(ds.zgroup.arrays,String(varname))
+function Base.haskey(ds::ZarrDataset, varname::SymbolOrString)
+    haskey(ds.zgroup.arrays, String(varname))
+end
 
-function CDM.variable(ds::ZarrDataset,varname::SymbolOrString)
+# CommonDataModel.jl interface methods
+
+CDM.name(v::ZarrDataset) = Zarr.zname(v.zgroup)
+function CDM.variable(ds::ZarrDataset, varname::SymbolOrString)
     zarray = ds.zgroup.arrays[String(varname)]
-    ZarrVariable{eltype(zarray),ndims(zarray),typeof(zarray),typeof(ds)}(zarray,ds)
+    ZarrVariable{eltype(zarray),ndims(zarray),typeof(zarray),typeof(ds)}(zarray, ds)
 end
-
 CDM.dimnames(ds::ZarrDataset) = Tuple(String.(keys(ds.dimensions)))
+CDM.dim(ds::ZarrDataset, dimname::SymbolOrString) = ds.dimensions[Symbol(dimname)]
+CDM.varnames(ds::ZarrDataset) = keys(ds.zgroup.arrays)
+CDM.attribnames(ds::ZarrDataset) = keys(ds.zgroup.attrs)
+CDM.attrib(ds::ZarrDataset, name::SymbolOrString) = ds.zgroup.attrs[String(name)]
 
 # function CDM.unlimited(ds::ZarrDataset)
 #     ul = ds.unlimited
@@ -18,67 +26,40 @@ CDM.dimnames(ds::ZarrDataset) = Tuple(String.(keys(ds.dimensions)))
 #     return ul
 # end
 
-function _dim(ds::ZarrDataset,dimname::SymbolOrString)
-    dimlen = get(ds.dimensions,Symbol(dimname),nothing)
-
-    if !isnothing(dimlen)
-        return dimlen
-    end
-
-    if ds.parentdataset !== nothing
-        return _dim(ds.parentdataset,dimname)
-    end
-
-    error("dimension $dimname is not defined")
-end
-
-CDM.dim(ds::ZarrDataset,dimname::SymbolOrString) = ds.dimensions[Symbol(dimname)]
-
-function CDM.defDim(ds::ZarrDataset,dimname::SymbolOrString,dimlen)
+function CDM.defDim(ds::ZarrDataset, dimname::SymbolOrString, dimlen)
     dn = Symbol(dimname)
-    @assert !haskey(ds.dimensions,dn)
+    @assert !haskey(ds.dimensions, dn)
     ds.dimensions[dn] = dimlen
 end
 
-CDM.varnames(ds::ZarrDataset) = keys(ds.zgroup.arrays)
-
-CDM.attribnames(ds::ZarrDataset) = keys(ds.zgroup.attrs)
-CDM.attrib(ds::ZarrDataset,name::SymbolOrString) = ds.zgroup.attrs[String(name)]
-
-function CDM.defAttrib(ds::ZarrDataset,name::SymbolOrString,value)
+function CDM.defAttrib(ds::ZarrDataset, name::SymbolOrString, value)
     @assert iswritable(ds)
     ds.zgroup.attrs[String(name)] = value
 
     storage = ds.zgroup.storage
     io = IOBuffer()
     JSON.print(io, ds.zgroup.attrs)
-    storage[ds.zgroup.path,".zattrs"] = take!(io)
+    storage[ds.zgroup.path, ".zattrs"] = take!(io)
 end
 
 # groups
-
-function CDM.defGroup(ds::ZarrDataset,groupname::SymbolOrString; attrib = Dict())
+function CDM.defGroup(ds::ZarrDataset, groupname::SymbolOrString; attrib=Dict())
     _attrib = Dict{String,Any}(attrib)
-    zg = zgroup(ds.zgroup,String(groupname),attrs = _attrib)
+    zg = zgroup(ds.zgroup, String(groupname); attrs=_attrib)
     dimensions = OrderedDict{Symbol,Int}()
-    return ZarrDataset(ds,zg,dimensions,ds.iswritable,ds.maskingvalue)
+    return ZarrDataset(ds, zg, dimensions, ds.iswritable, ds.maskingvalue)
 end
-
 CDM.groupnames(ds::ZarrDataset) = keys(ds.zgroup.groups)
-
-function CDM.group(ds::ZarrDataset,groupname::SymbolOrString)
+function CDM.group(ds::ZarrDataset, groupname::SymbolOrString)
     dimensions = OrderedDict{Symbol,Int}()
     zg = ds.zgroup.groups[String(groupname)]
-    return ZarrDataset(ds,zg,dimensions,ds.iswritable,ds.maskingvalue)
+    return ZarrDataset(ds, zg, dimensions, ds.iswritable, ds.maskingvalue)
 end
 
-
-
 CDM.parentdataset(ds::ZarrDataset) = ds.parentdataset
 CDM.iswritable(ds::ZarrDataset) = ds.iswritable
 CDM.maskingvalue(ds::ZarrDataset) = ds.maskingvalue
 
-
 """
     ds = ZarrDataset(url::AbstractString,mode = "r";
                      _omitcode = [404,403],
@@ -90,13 +71,17 @@ CDM.maskingvalue(ds::ZarrDataset) = ds.maskingvalue
 Open the zarr dataset at the url or path `url`. The mode can be `"r"` (read-only),
 `"w"` (write), or `"c"` (create). `ds` supports the API of the
 [JuliaGeo/CommonDataModel.jl](https://github.com/JuliaGeo/CommonDataModel.jl).
-The experimental `_omitcode` allows to define which HTTP error code should be used
-for missing chunks. For compatibility with python's Zarr, the HTTP error 403
-(permission denied) is also used to missing chunks in addition to 404 (not
-found).
 
-The parameter `maskingvalue` allows to define which special value should be used
-as replacement for fill values. The default is `missing`.
+# Keywords
+
+- `_omitcode`: *experimental*, allows to define which HTTP error code should be used
+    for missing chunks. For compatibility with python's Zarr, the HTTP error 403
+    (permission denied) is also used to missing chunks in addition to 404 (not
+    found). Default [404, 403].
+- `maskingvalue`: The parameter `maskingvalue` allows to define which special 
+    value should be used as replacement for fill values. The default is `missing`.
+    Defaults to `missing``.
+- `attrib`: Attributes, defualts to `Dict()`
 
 Example:
 
@@ -128,72 +113,80 @@ zos1 = ZarrDataset(url) do ds
 end # implicit call to close(ds)
 ```
 """
-function ZarrDataset(url::AbstractString, mode = "r";
-                     parentdataset = nothing,
-                     _omitcode = [404,403],
-                     maskingvalue = missing,
-                     attrib = Dict(),
-                     )
-
+function ZarrDataset(
+    url::AbstractString,
+    mode="r";
+    parentdataset=nothing,
+    _omitcode=[404, 403],
+    maskingvalue=missing,
+    attrib=Dict(),
+)
     dimensions = OrderedDict{Symbol,Int}()
 
     zg = if mode in ("w", "r")
         zg = Zarr.zopen(url, mode)
     elseif mode == "c"
         store = Zarr.DirectoryStore(url)
-        zg = zgroup(store, "", attrs = Dict{String,Any}(attrib))
+        zg = zgroup(store, ""; attrs=Dict{String,Any}(attrib))
     else
         throw(ArgumentError("mode must be \"r\", \"w\" or \"c\", got $mode"))
     end
     ZarrDataset(zg; mode, parentdataset, _omitcode, maskingvalue, attrib)
 end
 
-function ZarrDataset(store::Zarr.AbstractStore, mode = "r";
-                     parentdataset = nothing,
-                     _omitcode = [404,403],
-                     maskingvalue = missing,
-                     attrib = Dict(),
-                     )
-    return ZarrDataset(zopen(store, mode); mode, parentdataset, _omitcode, maskingvalue, attrib)
+function ZarrDataset(
+    store::Zarr.AbstractStore,
+    mode="r";
+    parentdataset=nothing,
+    _omitcode=[404, 403],
+    maskingvalue=missing,
+    attrib=Dict(),
+)
+    return ZarrDataset(
+        zopen(store, mode); mode, parentdataset, _omitcode, maskingvalue, attrib
+    )
 end
 
-function ZarrDataset(zg::Zarr.ZGroup;
-                     mode = "r",
-                     parentdataset = nothing,
-                     _omitcode = [404,403],
-                     maskingvalue = missing,
-                     attrib = Dict(),
-                     )
-
+function ZarrDataset(
+    zg::Zarr.ZGroup;
+    mode="r",
+    parentdataset=nothing,
+    _omitcode=[404, 403],
+    maskingvalue=missing,
+    attrib=Dict(),
+)
     dimensions = ZarrDatasets.OrderedDict{Symbol,Int}()
     iswritable = false
     if (zg.storage isa Zarr.HTTPStore) ||
         (zg.storage isa Zarr.ConsolidatedStore{Zarr.HTTPStore})
-            @debug "omit chunks on HTTP error" _omitcode
-            Zarr.missing_chunk_return_code!(zg.storage,_omitcode)
-        end
+        @debug "omit chunks on HTTP error" _omitcode
+        Zarr.missing_chunk_return_code!(zg.storage, _omitcode)
+    end
 
-        for (varname,zarray) in zg.arrays
-            for (dimname,dimlen) in zip(reverse(zarray.attrs["_ARRAY_DIMENSIONS"]),size(zarray))
-                dn = Symbol(dimname)
-                if haskey(dimensions,dn)
-                    @assert dimensions[dn] == dimlen
-                else
-                    dimensions[dn] = dimlen
-                end
+    for (varname, zarray) in zg.arrays
+        for (dimname, dimlen) in
+            zip(reverse(zarray.attrs["_ARRAY_DIMENSIONS"]), size(zarray))
+            dn = Symbol(dimname)
+            if haskey(dimensions, dn)
+                @assert dimensions[dn] == dimlen
+            else
+                dimensions[dn] = dimlen
+            end
         end
     end
 
-    return ZarrDataset(parentdataset, zg, dimensions, mode == "r" ? false : zg.writeable, maskingvalue)
-
+    return ZarrDataset(
+        parentdataset, zg, dimensions, mode == "r" ? false : zg.writeable, maskingvalue
+    )
 end
 
+function ZarrDataset(
+    fnames::AbstractArray{<:AbstractString,N}, args...; kwargs...
+) where {N}
+    MFDataset(ZarrDataset, fnames, args...; kwargs...)
+end
 
-ZarrDataset(fnames::AbstractArray{<:AbstractString,N}, args...; kwargs...) where N =
-    MFDataset(ZarrDataset,fnames, args...; kwargs...)
-
-
-function ZarrDataset(f::Function,args...; kwargs...)
+function ZarrDataset(f::Function, args...; kwargs...)
     ds = ZarrDataset(args...; kwargs...)
     try
         f(ds)
@@ -202,7 +195,18 @@ function ZarrDataset(f::Function,args...; kwargs...)
     end
 end
 
-export ZarrDataset
-export defDim
-export defVar
-export defGroup
+# Utility functions
+
+function _dim(ds::ZarrDataset, dimname::SymbolOrString)
+    dimlen = get(ds.dimensions, Symbol(dimname), nothing)
+
+    if !isnothing(dimlen)
+        return dimlen
+    end
+
+    if ds.parentdataset !== nothing
+        return _dim(ds.parentdataset, dimname)
+    end
+
+    error("dimension $dimname is not defined")
+end

src/types.jl

@@ -1,10 +1,21 @@
 
-struct ZarrVariable{T,N,TZA <: AbstractArray{T,N},TZG} <: CDM.AbstractVariable{T,N}
+"""
+    ZarrVariable <: CDM.AbstractVariable
+
+    ZarrVariable(zarray, parentdataset)
+
+A CommonDataModel.jl compatible zarr variable.
+
+Usually generated with `CommonDataModel.defvar(::ZarrDataset, args...; kw...)`
+"""
+struct ZarrVariable{T,N,TZA<:AbstractArray{T,N},TZG} <: CDM.AbstractVariable{T,N}
     zarray::TZA
     parentdataset::TZG
 end
 
-struct ZarrDataset{TDS <: Union{CDM.AbstractDataset,Nothing},Tmaskingvalue,TZ} <: CDM.AbstractDataset
+# Documented in dataset.jl
+struct ZarrDataset{TDS<:Union{CDM.AbstractDataset,Nothing},Tmaskingvalue,TZ} <:
+       CDM.AbstractDataset
     parentdataset::TDS
     zgroup::TZ
     dimensions::OrderedDict{Symbol,Int}