Round-trip reinterpret of all isbits types (#47116)

Hiding padding bytes in the process, to avoid undefined behavior if
those are observed.

Co-authored-by: Andy Ferris <[email protected]>
Co-authored-by: Jameson Nash <[email protected]>
Co-authored-by: Sukera <[email protected]>

Author: 4 people

base/reinterpretarray.jl

@@ -651,8 +651,8 @@ end
 
 # Padding
 struct Padding
-    offset::Int
-    size::Int
+    offset::Int # 0-indexed offset of the next valid byte; sizeof(T) indicates trailing padding
+    size::Int   # bytes of padding before a valid byte
 end
 function intersect(p1::Padding, p2::Padding)
     start = max(p1.offset, p2.offset)
@@ -696,20 +696,24 @@ function iterate(cp::CyclePadding, state::Tuple)
 end
 
 """
-    Compute the location of padding in a type.
+    Compute the location of padding in an isbits datatype. Recursive over the fields of that type.
 """
-function padding(T)
-    padding = Padding[]
-    last_end::Int = 0
+@assume_effects :foldable function padding(T::DataType, baseoffset::Int = 0)
+    pads = Padding[]
+    last_end::Int = baseoffset
     for i = 1:fieldcount(T)
-        offset = fieldoffset(T, i)
+        offset = baseoffset + Int(fieldoffset(T, i))
         fT = fieldtype(T, i)
+        append!(pads, padding(fT, offset))
         if offset != last_end
-            push!(padding, Padding(offset, offset-last_end))
+            push!(pads, Padding(offset, offset-last_end))
         end
         last_end = offset + sizeof(fT)
     end
-    padding
+    if 0 < last_end - baseoffset < sizeof(T)
+        push!(pads, Padding(baseoffset + sizeof(T), sizeof(T) - last_end + baseoffset))
+    end
+    return Core.svec(pads...)
 end
 
 function CyclePadding(T::DataType)
@@ -748,6 +752,124 @@ end
     return true
 end
 
+@assume_effects :foldable function struct_subpadding(::Type{Out}, ::Type{In}) where {Out, In}
+    padding(Out) == padding(In)
+end
+
+@assume_effects :foldable function packedsize(::Type{T}) where T
+    pads = padding(T)
+    return sizeof(T) - sum((p.size for p ∈ pads), init = 0)
+end
+
+@assume_effects :foldable ispacked(::Type{T}) where T = isempty(padding(T))
+
+function _copytopacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In}
+    writeoffset = 0
+    for i ∈ 1:fieldcount(In)
+        readoffset = fieldoffset(In, i)
+        fT = fieldtype(In, i)
+        if ispacked(fT)
+            readsize = sizeof(fT)
+            memcpy(ptr_out + writeoffset, ptr_in + readoffset, readsize)
+            writeoffset += readsize
+        else # nested padded type
+            _copytopacked!(ptr_out + writeoffset, Ptr{fT}(ptr_in + readoffset))
+            writeoffset += packedsize(fT)
+        end
+    end
+end
+
+function _copyfrompacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In}
+    readoffset = 0
+    for i ∈ 1:fieldcount(Out)
+        writeoffset = fieldoffset(Out, i)
+        fT = fieldtype(Out, i)
+        if ispacked(fT)
+            writesize = sizeof(fT)
+            memcpy(ptr_out + writeoffset, ptr_in + readoffset, writesize)
+            readoffset += writesize
+        else # nested padded type
+            _copyfrompacked!(Ptr{fT}(ptr_out + writeoffset), ptr_in + readoffset)
+            readoffset += packedsize(fT)
+        end
+    end
+end
+
+"""
+    reinterpret(::Type{Out}, x::In)
+
+Reinterpret the valid non-padding bytes of an isbits value `x` as isbits type `Out`.
+
+Both types must have the same amount of non-padding bytes. This operation is guaranteed
+to be reversible.
+
+```jldoctest
+julia> reinterpret(NTuple{2, UInt8}, 0x1234)
+(0x34, 0x12)
+
+julia> reinterpret(UInt16, (0x34, 0x12))
+0x1234
+
+julia> reinterpret(Tuple{UInt16, UInt8}, (0x01, 0x0203))
+(0x0301, 0x02)
+```
+
+!!! warning
+
+    Use caution if some combinations of bits in `Out` are not considered valid and would
+    otherwise be prevented by the type's constructors and methods. Unexpected behavior
+    may result without additional validation.
+"""
+@inline function reinterpret(::Type{Out}, x::In) where {Out, In}
+    isbitstype(Out) || throw(ArgumentError("Target type for `reinterpret` must be isbits"))
+    isbitstype(In) || throw(ArgumentError("Source type for `reinterpret` must be isbits"))
+    if isprimitivetype(Out) && isprimitivetype(In)
+        outsize = sizeof(Out)
+        insize = sizeof(In)
+        outsize == insize ||
+            throw(ArgumentError("Sizes of types $Out and $In do not match; got $outsize \
+                and $insize, respectively."))
+        return bitcast(Out, x)
+    end
+    inpackedsize = packedsize(In)
+    outpackedsize = packedsize(Out)
+    inpackedsize == outpackedsize ||
+        throw(ArgumentError("Packed sizes of types $Out and $In do not match; got $outpackedsize \
+            and $inpackedsize, respectively."))
+    in = Ref{In}(x)
+    out = Ref{Out}()
+    if struct_subpadding(Out, In)
+        # if packed the same, just copy
+        GC.@preserve in out begin
+            ptr_in = unsafe_convert(Ptr{In}, in)
+            ptr_out = unsafe_convert(Ptr{Out}, out)
+            memcpy(ptr_out, ptr_in, sizeof(Out))
+        end
+        return out[]
+    else
+        # mismatched padding
+        GC.@preserve in out begin
+            ptr_in = unsafe_convert(Ptr{In}, in)
+            ptr_out = unsafe_convert(Ptr{Out}, out)
+
+            if fieldcount(In) > 0 && ispacked(Out)
+                _copytopacked!(ptr_out, ptr_in)
+            elseif fieldcount(Out) > 0 && ispacked(In)
+                _copyfrompacked!(ptr_out, ptr_in)
+            else
+                packed = Ref{NTuple{inpackedsize, UInt8}}()
+                GC.@preserve packed begin
+                    ptr_packed = unsafe_convert(Ptr{NTuple{inpackedsize, UInt8}}, packed)
+                    _copytopacked!(ptr_packed, ptr_in)
+                    _copyfrompacked!(ptr_out, ptr_packed)
+                end
+            end
+        end
+        return out[]
+    end
+end
+
+
 # Reductions with IndexSCartesian2
 
 function _mapreduce(f::F, op::OP, style::IndexSCartesian2{K}, A::AbstractArrayOrBroadcasted) where {F,OP,K}

test/core.jl

@@ -1897,7 +1897,7 @@ function f4528(A, B)
     end
 end
 @test f4528(false, Int32(12)) === nothing
-@test_throws ErrorException f4528(true, Int32(12))
+@test_throws ArgumentError f4528(true, Int32(12))
 
 # issue #4518
 f4518(x, y::Union{Int32,Int64}) = 0

test/numbers.jl

@@ -2216,13 +2216,11 @@ end
     @test round(Int16, -32768.1) === Int16(-32768)
 end
 # issue #7508
-@test_throws ErrorException reinterpret(Int, 0x01)
+@test_throws ArgumentError reinterpret(Int, 0x01)
 
 @testset "issue #12832" begin
-    @test_throws ErrorException reinterpret(Float64, Complex{Int64}(1))
-    @test_throws ErrorException reinterpret(Float64, ComplexF32(1))
-    @test_throws ErrorException reinterpret(ComplexF32, Float64(1))
-    @test_throws ErrorException reinterpret(Int32, false)
+    @test_throws ArgumentError reinterpret(Float64, Complex{Int64}(1))
+    @test_throws ArgumentError reinterpret(Int32, false)
 end
 # issue #41
 ndigf(n) = Float64(log(Float32(n)))

test/reinterpretarray.jl

@@ -450,10 +450,10 @@ end
         SomeSingleton(x) = new()
     end
 
-    @test_throws ErrorException reinterpret(Int, nothing)
-    @test_throws ErrorException reinterpret(Missing, 3)
-    @test_throws ErrorException reinterpret(Missing, NotASingleton())
-    @test_throws ErrorException reinterpret(NotASingleton, ())
+    @test_throws ArgumentError reinterpret(Int, nothing)
+    @test_throws ArgumentError reinterpret(Missing, 3)
+    @test_throws ArgumentError reinterpret(Missing, NotASingleton())
+    @test_throws ArgumentError reinterpret(NotASingleton, ())
 
     @test_throws ArgumentError reinterpret(NotASingleton, fill(nothing, ()))
     @test_throws ArgumentError reinterpret(reshape, NotASingleton, fill(missing, 3))
@@ -513,3 +513,25 @@ end
     @test setindex!(x, SomeSingleton(:), 3, 5) == x2
     @test_throws MethodError x[2,4] = nothing
 end
+
+# reinterpret of arbitrary bitstypes
+@testset "Reinterpret arbitrary bitstypes" begin
+    struct Bytes15
+        a::Int8
+        b::Int16
+        c::Int32
+        d::Int64
+    end
+
+    @test reinterpret(Float64, ComplexF32(1, 1)) === 0.007812501848093234
+    @test reinterpret(ComplexF32, 0.007812501848093234) === ComplexF32(1, 1)
+    @test reinterpret(Tuple{Float64, Float64}, ComplexF64(1, 1)) === (1.0, 1.0)
+    @test reinterpret(ComplexF64, (1.0, 1.0)) === ComplexF64(1, 1)
+    @test reinterpret(Tuple{Int8, Int16, Int32, Int64}, (Int64(1), Int32(2), Int16(3), Int8(4))) === (Int8(1), Int16(0), Int32(0), 288233674686595584)
+    @test reinterpret(Tuple{Int8, Int16, Tuple{Int32, Int64}}, (Int64(1), Int32(2), Int16(3), Int8(4))) === (Int8(1), Int16(0), (Int32(0), 288233674686595584))
+    @test reinterpret(Tuple{Int64, Int32, Int16, Int8}, (Int8(1), Int16(0), (Int32(0), 288233674686595584))) === (Int64(1), Int32(2), Int16(3), Int8(4))
+    @test reinterpret(Tuple{Int8, Int16, Int32, Int64}, Bytes15(Int8(1), Int16(2), Int32(3), Int64(4))) === (Int8(1), Int16(2), Int32(3), Int64(4))
+    @test reinterpret(Bytes15, (Int8(1), Int16(2), Int32(3), Int64(4))) == Bytes15(Int8(1), Int16(2), Int32(3), Int64(4))
+
+    @test_throws ArgumentError reinterpret(Tuple{Int32, Int64}, (Int16(1), Int64(4)))
+end