Skip to content

Ensure that aggregation is consistent regardless of data alignment #106166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 13, 2024
Merged

Ensure that aggregation is consistent regardless of data alignment #106166

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
23ed7eb
Ensure that aggregation is consistent regardless of data alignment
tannergooding Aug 8, 2024
b3d460e
Ensure we handle for all aggregation helpers
tannergooding Aug 9, 2024
864920d
Ensure we don't process beg twice
tannergooding Aug 9, 2024
238c947
Ensure that we properly track in the case we can't align
tannergooding Aug 12, 2024
23c043c
Add missing semicolon
tannergooding Aug 12, 2024
1f8f4a8
Fix the handling on .NET Framework
tannergooding Aug 12, 2024
211d70f
Ensure yptr on .NET Framework is incremented as well
tannergooding Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions ...nsors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IAggregationOperator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1227,9 +1227,12 @@ static T Vectorized128(ref T xRef, ref T yRef, nuint remainder)

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// can never achieve the required SIMD alignment. This cannot be done for
// float or double since that changes how results compound together.

bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0;
bool canAlign = (typeof(T) != typeof(float)) &&
(typeof(T) != typeof(double)) &&
((nuint)xPtr % (nuint)sizeof(T)) == 0;

if (canAlign)
{
Expand Down Expand Up @@ -1418,9 +1421,12 @@ static T Vectorized256(ref T xRef, ref T yRef, nuint remainder)

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// can never achieve the required SIMD alignment. This cannot be done for
// float or double since that changes how results compound together.

bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0;
bool canAlign = (typeof(T) != typeof(float)) &&
(typeof(T) != typeof(double)) &&
((nuint)xPtr % (nuint)sizeof(T)) == 0;

if (canAlign)
{
Expand Down Expand Up @@ -1609,9 +1615,12 @@ static T Vectorized512(ref T xRef, ref T yRef, nuint remainder)

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// can never achieve the required SIMD alignment. This cannot be done for
// float or double since that changes how results compound together.

bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0;
bool canAlign = (typeof(T) != typeof(float)) &&
(typeof(T) != typeof(double)) &&
((nuint)xPtr % (nuint)sizeof(T)) == 0;

if (canAlign)
{
Expand Down
31 changes: 4 additions & 27 deletions ...cs.Tensors/src/System/Numerics/Tensors/netstandard/TensorPrimitives.Single.netstandard.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,6 @@ static float Vectorized(ref float xRef, ref float yRef, nuint remainder, TBinary
Vector<float> end = binaryOp.Invoke(AsVector(ref xRef, remainder - (uint)(Vector<float>.Count)),
AsVector(ref yRef, remainder - (uint)(Vector<float>.Count)));

nuint misalignment = 0;

if (remainder > (uint)(Vector<float>.Count * 8))
{
// Pinning is cheap and will be short lived for small inputs and unlikely to be impactful
Expand All @@ -480,29 +478,9 @@ static float Vectorized(ref float xRef, ref float yRef, nuint remainder, TBinary
float* xPtr = px;
float* yPtr = py;

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.

bool canAlign = ((nuint)(xPtr) % sizeof(float)) == 0;

if (canAlign)
{
// Compute by how many elements we're misaligned and adjust the pointers accordingly
//
// Noting that we are only actually aligning dPtr. This is because unaligned stores
// are more expensive than unaligned loads and aligning both is significantly more
// complex.

misalignment = ((uint)(sizeof(Vector<float>)) - ((nuint)(xPtr) % (uint)(sizeof(Vector<float>)))) / sizeof(float);

xPtr += misalignment;
yPtr += misalignment;

Debug.Assert(((nuint)(xPtr) % (uint)(sizeof(Vector<float>))) == 0);

remainder -= misalignment;
}
// Unlike many other vectorization algorithms, we cannot align for aggregation
// because that changes how results compound together and can cause a significant
// difference in the output.

Vector<float> vector1;
Vector<float> vector2;
Expand Down Expand Up @@ -564,7 +542,6 @@ static float Vectorized(ref float xRef, ref float yRef, nuint remainder, TBinary
// Store the first block. Handling this separately simplifies the latter code as we know
// they come after and so we can relegate it to full blocks or the trailing elements

beg = Vector.ConditionalSelect(CreateAlignmentMaskSingleVector((int)(misalignment)), beg, new Vector<float>(aggregationOp.IdentityValue));
vresult = aggregationOp.Invoke(vresult, beg);

// Process the remaining [0, Count * 7] elements via a jump table
Expand All @@ -575,7 +552,7 @@ static float Vectorized(ref float xRef, ref float yRef, nuint remainder, TBinary

nuint blocks = remainder / (nuint)(Vector<float>.Count);
nuint trailing = remainder - (blocks * (nuint)(Vector<float>.Count));
blocks -= (misalignment == 0) ? 1u : 0u;
blocks -= 1u;
remainder -= trailing;

switch (blocks)
Expand Down