Repository navigation
[API Proposal]: Arm64: FEAT_SVE: firstfaulting #94004
Description
namespace System.Runtime.Intrinsics.Arm;
/// VectorT Summary
public abstract partial class Sve : AdvSimd /// Feature: FEAT_SVE Category: firstfaulting
{
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorByteZeroExtendFirstFaulting(Vector<T> mask, Vector<T2> bases); // LDFF1B
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorByteZeroExtendFirstFaulting(Vector<T> mask, Vector<T> bases); // LDFF1B
/// T: int, uint, long, ulong
public static unsafe Vector<T> GatherVectorByteZeroExtendFirstFaulting(Vector<T> mask, byte* base, Vector<T> offsets); // LDFF1B
/// T: [uint, int], [int, uint], [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorByteZeroExtendFirstFaulting(Vector<T> mask, byte* base, Vector<T2> offsets); // LDFF1B
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorByteZeroExtendFirstFaulting(Vector<T> mask, Vector<T2> bases, long offset); // LDFF1B
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorByteZeroExtendFirstFaulting(Vector<T> mask, Vector<T> bases, long offset); // LDFF1B
/// T: [float, uint], [int, uint], [double, ulong], [long, ulong]
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, Vector<T2> bases); // LDFF1W or LDFF1D
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, Vector<T> bases); // LDFF1W or LDFF1D
/// T: [float, int], [uint, int], [float, uint], [int, uint], [double, long], [ulong, long], [double, ulong], [long, ulong]
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, T* base, Vector<T2> offsets); // LDFF1W or LDFF1D
/// T: int, uint, long, ulong
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, T* base, Vector<T> offsets); // LDFF1W or LDFF1D
/// T: [float, int], [uint, int], [float, uint], [int, uint], [double, long], [ulong, long], [double, ulong], [long, ulong]
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, T* base, Vector<T2> indices); // LDFF1W or LDFF1D
/// T: int, uint, long, ulong
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, T* base, Vector<T> indices); // LDFF1W or LDFF1D
/// T: [float, uint], [int, uint], [double, ulong], [long, ulong]
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, Vector<T2> bases, long offset); // LDFF1W or LDFF1D
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, Vector<T> bases, long offset); // LDFF1W or LDFF1D
/// T: [float, uint], [int, uint], [double, ulong], [long, ulong]
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, Vector<T2> bases, long index); // LDFF1W or LDFF1D
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorFirstFaulting(Vector<T> mask, Vector<T> bases, long index); // LDFF1W or LDFF1D
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, Vector<T2> bases); // LDFF1SH
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, Vector<T> bases); // LDFF1SH
/// T: int, uint, long, ulong
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, short* base, Vector<T> offsets); // LDFF1SH
/// T: [uint, int], [int, uint], [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, short* base, Vector<T2> offsets); // LDFF1SH
/// T: int, uint, long, ulong
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, short* base, Vector<T> indices); // LDFF1SH
/// T: [uint, int], [int, uint], [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, short* base, Vector<T2> indices); // LDFF1SH
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, Vector<T2> bases, long offset); // LDFF1SH
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, Vector<T> bases, long offset); // LDFF1SH
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, Vector<T2> bases, long index); // LDFF1SH
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorInt16SignExtendFirstFaulting(Vector<T> mask, Vector<T> bases, long index); // LDFF1SH
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, Vector<T2> bases); // LDFF1H
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, Vector<T> bases); // LDFF1H
/// T: int, uint, long, ulong
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, ushort* base, Vector<T> offsets); // LDFF1H
/// T: [uint, int], [int, uint], [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, ushort* base, Vector<T2> offsets); // LDFF1H
/// T: int, uint, long, ulong
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, ushort* base, Vector<T> indices); // LDFF1H
/// T: [uint, int], [int, uint], [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, ushort* base, Vector<T2> indices); // LDFF1H
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, Vector<T2> bases, long offset); // LDFF1H
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, Vector<T> bases, long offset); // LDFF1H
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, Vector<T2> bases, long index); // LDFF1H
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorInt16ZeroExtendFirstFaulting(Vector<T> mask, Vector<T> bases, long index); // LDFF1H
public static unsafe Vector<long> GatherVectorInt32SignExtendFirstFaulting(Vector<long> mask, Vector<ulong> bases); // LDFF1SW
public static unsafe Vector<ulong> GatherVectorInt32SignExtendFirstFaulting(Vector<ulong> mask, Vector<ulong> bases); // LDFF1SW
/// T: long, ulong
public static unsafe Vector<T> GatherVectorInt32SignExtendFirstFaulting(Vector<T> mask, int* base, Vector<T> offsets); // LDFF1SW
/// T: [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorInt32SignExtendFirstFaulting(Vector<T> mask, int* base, Vector<T2> offsets); // LDFF1SW
/// T: long, ulong
public static unsafe Vector<T> GatherVectorInt32SignExtendFirstFaulting(Vector<T> mask, int* base, Vector<T> indices); // LDFF1SW
/// T: [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorInt32SignExtendFirstFaulting(Vector<T> mask, int* base, Vector<T2> indices); // LDFF1SW
public static unsafe Vector<long> GatherVectorInt32SignExtendFirstFaulting(Vector<long> mask, Vector<ulong> bases, long offset); // LDFF1SW
public static unsafe Vector<ulong> GatherVectorInt32SignExtendFirstFaulting(Vector<ulong> mask, Vector<ulong> bases, long offset); // LDFF1SW
public static unsafe Vector<long> GatherVectorInt32SignExtendFirstFaulting(Vector<long> mask, Vector<ulong> bases, long index); // LDFF1SW
public static unsafe Vector<ulong> GatherVectorInt32SignExtendFirstFaulting(Vector<ulong> mask, Vector<ulong> bases, long index); // LDFF1SW
public static unsafe Vector<long> GatherVectorInt32ZeroExtendFirstFaulting(Vector<long> mask, Vector<ulong> bases); // LDFF1W
public static unsafe Vector<ulong> GatherVectorInt32ZeroExtendFirstFaulting(Vector<ulong> mask, Vector<ulong> bases); // LDFF1W
/// T: long, ulong
public static unsafe Vector<T> GatherVectorInt32ZeroExtendFirstFaulting(Vector<T> mask, uint* base, Vector<T> offsets); // LDFF1W
/// T: [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorInt32ZeroExtendFirstFaulting(Vector<T> mask, uint* base, Vector<T2> offsets); // LDFF1W
/// T: long, ulong
public static unsafe Vector<T> GatherVectorInt32ZeroExtendFirstFaulting(Vector<T> mask, uint* base, Vector<T> indices); // LDFF1W
/// T: [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorInt32ZeroExtendFirstFaulting(Vector<T> mask, uint* base, Vector<T2> indices); // LDFF1W
public static unsafe Vector<long> GatherVectorInt32ZeroExtendFirstFaulting(Vector<long> mask, Vector<ulong> bases, long offset); // LDFF1W
public static unsafe Vector<ulong> GatherVectorInt32ZeroExtendFirstFaulting(Vector<ulong> mask, Vector<ulong> bases, long offset); // LDFF1W
public static unsafe Vector<long> GatherVectorInt32ZeroExtendFirstFaulting(Vector<long> mask, Vector<ulong> bases, long index); // LDFF1W
public static unsafe Vector<ulong> GatherVectorInt32ZeroExtendFirstFaulting(Vector<ulong> mask, Vector<ulong> bases, long index); // LDFF1W
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorSByteSignExtendFirstFaulting(Vector<T> mask, Vector<T2> bases); // LDFF1SB
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorSByteSignExtendFirstFaulting(Vector<T> mask, Vector<T> bases); // LDFF1SB
/// T: int, uint, long, ulong
public static unsafe Vector<T> GatherVectorSByteSignExtendFirstFaulting(Vector<T> mask, sbyte* base, Vector<T> offsets); // LDFF1SB
/// T: [uint, int], [int, uint], [ulong, long], [long, ulong]
public static unsafe Vector<T> GatherVectorSByteSignExtendFirstFaulting(Vector<T> mask, sbyte* base, Vector<T2> offsets); // LDFF1SB
/// T: [int, uint], [long, ulong]
public static unsafe Vector<T> GatherVectorSByteSignExtendFirstFaulting(Vector<T> mask, Vector<T2> bases, long offset); // LDFF1SB
/// T: uint, ulong
public static unsafe Vector<T> GatherVectorSByteSignExtendFirstFaulting(Vector<T> mask, Vector<T> bases, long offset); // LDFF1SB
/// T: sbyte, short, int, long, byte, ushort, uint, ulong
public static unsafe Vector<T> GetFFR(); // RDFFR // predicated
/// T: short, int, long, ushort, uint, ulong
public static unsafe Vector<T> LoadVectorByteZeroExtendFirstFaulting(byte* address); // LDFF1B // predicated
/// T: float, double, sbyte, short, int, long, byte, ushort, uint, ulong
public static unsafe Vector<T> LoadVectorFirstFaulting(T* address); // LDFF1W or LDFF1D or LDFF1B or LDFF1H // predicated
/// T: int, long, uint, ulong
public static unsafe Vector<T> LoadVectorInt16SignExtendFirstFaulting(short* address); // LDFF1SH // predicated
/// T: int, long, uint, ulong
public static unsafe Vector<T> LoadVectorInt16ZeroExtendFirstFaulting(ushort* address); // LDFF1H // predicated
/// T: long, ulong
public static unsafe Vector<T> LoadVectorInt32SignExtendFirstFaulting(int* address); // LDFF1SW // predicated
/// T: long, ulong
public static unsafe Vector<T> LoadVectorInt32ZeroExtendFirstFaulting(uint* address); // LDFF1W // predicated
/// T: short, int, long, ushort, uint, ulong
public static unsafe Vector<T> LoadVectorSByteSignExtendFirstFaulting(sbyte* address); // LDFF1SB // predicated
public static unsafe void SetFFR(); // SETFFR
/// T: sbyte, short, int, long, byte, ushort, uint, ulong
public static unsafe void WriteFFR(Vector<T> value); // WRFFR
/// total method signatures: 72
}First Faulting Loads
Each First Faulting LoadVector method matches a non-faulting version in #94006.
Loads a vector element by element. If loading that element would cause a fault, then stop the load and set the first faulting register for each element that completed. If no elements fault, then the load completes as per the non-faulting version.
After calling a first faulting load, the first faulting mask should always be read to determine if the load completed.
GetFFR(), SetFFR() and WriteFFR() are used to access the ffr mask for the last first faulting load.
// Look for the 0 at the end of the buffer
long strlen(byte *buffer)
{
long i = 0;
Sve.SetFFR();
while (1)
{
// Load data from the buffer. maskfrr contains the elements that were loaded.
Vector<byte> data = Sve.LoadVectorFirstFaulting(buffer + i);
Vector<byte> maskfrr = Sve.GetFFR(all);
// Look for zeros in the loaded data
Vector<byte> zeromask = Sve.ConditionalSelect(maskfrr, Sve.CompareEquals(data, 0), Vector<byte>.Zero);
if (Sve.ConditionalSelect(Sve.MaskTestAnyTrue(zeromask))) {
// There was a zero in the loaded data. Increment up to the zero and exit the loop
zeromask = Sve.BreakBefore(zeromask);
i += Sve.GetActiveElementCount(zeromask);
break;
} else if (Sve.MaskTestLastTrue(maskffr)) {
// Final bit in the ffr mask is set, therefore the load completed. Increment and continue.
i += Sve.Count8BitElements();
} else {
// The load faulted. Increment by the number of loaded elements and continue.
Sve.SetFFR();
i += Sve.GetActiveElementCount(maskfrr);
}
}
return i;
}
Activity
a74nh commentedon Oct 26, 2023
a74nh commentedon Oct 26, 2023
a74nh commentedon Oct 26, 2023
This contributes to #93095
It covers instructions in FEAT_SVE related to first faulting loads.
This list was auto generated from the C ACLE for SVE, and is in three parts:
The methods list reduced down to Vector versions. All possible varaints of T are given above the method.
The complete list of all methods. The corresponding ACLE methods and SVE instructions are given above the method.
All rejected ACLE methods. These are methods we have agreed that do not need including in C#.
Where possible, existing C# naming conventions have been matched.
Many of the C functions include predicate argument(s), of type svbool_t as the first argument. These are missing from the C# method. It is expected that the Jit will create predicates where required, or combine with uses of conditionalSelect(). For more discussion see #88140 comment.
tannergooding commentedon Oct 26, 2023
Same general comments as the
Gather/LoadandScatter/StoreAPI proposals.These probably need a small explanation of how they work and what
SetFFRandWriteFFRdo, how they are expected to be used, and the implications of a user not resetting the FFR when done.a74nh commentedon Oct 31, 2023
I've taken an existing C strlen() in SVE, and rewritten it in C#. Both to show the use of FFR and to check it works with conditional selects.
10 remaining items
tannergooding commentedon Feb 8, 2024
Fixed the missing
Ua74nh commentedon Feb 9, 2024
After looking at the gather section, I think these are wrong in first faulting:
GatherVector*WithByteOffsetZeroExtendFirstFaultingneeds removingintanduintversionsLoadVector*ExtendFirstFaultingMy scripting thinks it should now look like:
(collapsed section)
tannergooding commentedon Feb 9, 2024
I expect we are going to need to manually validate the correct combinations as they are added. While the tool is convenient for getting a general API shape emitted, I believe it is also missing some nuance in places and it makes it hard to know how correct we are or not.
For example,
LDFF1Bhas seemingly 3 versions:LDFF1B (scalar plus scalar)LDFF1B (scalar plus vector)LDFF1B (vector plus immediate)LDFF1B (scalar plus scalar)The general signature is
LDFF1B { <Zt>.* }, <Pg>/Z, [<Xn|SP>{, <Xm>}]where<Zt>- The register to merge with, available viaConditionalSelect(mask, GatherVectorByteFirstFaulting(mask, address), Zt)<Pg>-mask<Xn|SP>-address<Xm>-offsetIt has encodings for
8,16,32, and64-bit elements. It functionally looks like:LDFF1B (scalar plus vector)The general signature is
LDFF1B { <Zt>.* }, <Pg>/Z, [<Xn|SP>, <Zm>.*, <mod>]where<Zt>- The register to merge with, available viaConditionalSelect(mask, GatherVectorByteFirstFaulting(mask, address, offsets), Zt)<Pg>-mask<Xn|SP>-address<Zm>-offsetsmod-0 for UXTW,1 for SXTW(indicating whetheroffsetsis signed or unsigned`)It has encodings for
32-bit unpacked unscaled offset,32-bit unscaled offset, and64-bit unscaled offset. It functionally looks like:I do not see an encoding that supports loading an
intusinglongoffsets.LDFF1B (vector plus immediate)The general signature is
LDFF1B { <Zt>.* }, <Pg>/Z, [<Zn>.*{, #<imm>}]where<Zt>- The register to merge with, available viaConditionalSelect(mask, GatherVectorByteFirstFaulting(mask, address), Zt)<Pg>-mask<Zn>-addresses<imm>-offset(must be0-31, inclusive)It has encodings for
32and64-bit elements. It functionally looks like:I do not see an encoding that supports using 64-bit addresses for zero-extending to 32-bit results.
Additional notes
All three of these forms have the comment:
So, depending on the desired shape for streaming mode, we may need to put these under a nested class indicating whether or not they are supported.
For
LDFF1Bin particular, since we are loading frombyte* address,indicesandoffsetare the same.Once we get to
LDFF1Hor larger we get the namesGatherVectorInt16FirstFaulting,GatherVectorUInt16ZeroExtendFirstFaulting, andGatherVectorUInt16FirstFaulting:scalar plus scalaroverload only allows forindices(it always doesbits(64) addr = base + (UInt(offset) + e) * mbytesscalar plus vectoroverload gets some new encodings32-bit scaled offset- usesindicesrather thanoffsets, supported forT: int, uint,T2: int, uint32-bit unpacked scaled offset- usesindicesrather thanoffsets, supported forT: long, ulong,T2: int, uint64-bit scaled offset- usesindicesrather thanoffsets, supported forT: long, ulong,T2: long, ulongvector plus immediateoverload adjusts<imm>to be a multiple of 2 in the range0-62, inclusiveAnd so on....
So, for
LDDFF1HI think that'd give us:We can expose
nuint offset = 0for thescalar plus scalarversions as it avoids additional overloads while still providing the convenience of a direct scalar (not forcing users to doVector<T2>.Zero)a74nh commentedon Feb 13, 2024
Agreed. The tool is still useful in spotting inconsistencies and it's being used to auto generate the tables and templates and full expanded api etc.
This one isn't a gather (it's just a single address)
So they should all just be
LoadVectorByteZeroExtendFirstFaulting(), which we have:Looking in the rejected pile, there are versions with an offset:
I thought in the reviews we decided to drop any addresses with an offset? Happy to re-include them.
These versions aren't valid. T and T2 must have the same size: https://docsmirror.github.io/A64/2023-09/ldff1b_z_p_bz.html
We could support it, but it would break down to two instructions (a sign extend of the offsets, then the gather load). So, I'd recommend not having it.
Again, T and T2 must be the same size:
https://docsmirror.github.io/A64/2023-09/ldff1b_z_p_ai.html
This becomes easier if the decision is to not support any SVE instructions while SME is enabled. I need to figure out if that's viable option though.
tannergooding commentedon Feb 13, 2024
Ah right. I read it initially and then didn't actually do it right when writing it down. Fixed
The consideration is we don't want to provide unnecessary API overloads when there is a trivially recognizable pattern. This is primarily because it reduces the number of APIs needed by several hundred across all of SVE/SVE2.
However, if there is a way to trivially support both via things like
optional parametersinstead, then we should feel free to propose that. This is because it allows us to do something likeLoadVectorByteZeroExtendFirstFaulting(Vector<T> mask, byte* address, nuint offset = 0)and cover both the offset provided and no offset provided scenario with a single API. So we provide the convenience without exploding the API surface.I think we're both "right" here. This one is a little confusing and it comes down to how we would want to expose the encoding to the user.
The page there shows that for
32-bit unpacked unscaled offsettheesize == 64(element size),msize == 8(memory size), andoffs_size == 32(offset size). -- This is in contrast to32-bit unscaled offsetwhereesize == 32,msize == 8, andoffs_size == 32.In the
Operation for all encodingsit then shows we useesizefor accessing themask. Foroffsetwe then useesizeto determine which element to access, but then only read the loweroffs_sizebits of eachesizeelement. Which is to say, it effectively takesVector<int>but in anevenlayout.Giving user access to that particular encoding of the instruction is then "complicated". We would either need the signature to take
Vector<int>and have users understand that it takes it in anevenlayout. Or we'd need a separately named API that takesVector<long>and where users understand it is zero/sign-extending the lower 32-bits, not the fulllong/ulong.Given that it is taking only the lower 32-bits (long) or even elements (int) (really depending on how you look at the API), it's not something we can trivially pattern match outside of constant inputs. However, for constant inputs, there isn't really an optimization to make since you need the same number of bits regardless.
Are you suggesting that it would duplicate the surface area for APIs like
Addin both SVE and SME?a74nh commentedon Feb 14, 2024
This seems reasonable. We'll have to revisit this for all the load and store APIs then, as I think most have have base+offset options.
You're right, it is the top one of the three encodings on that page.
Interesting, C does not have this as a function.
On https://dougallj.github.io/asil/ you can see that
LDFF1B { Zt.D }, Pg/Z, [Xn, Zm.D, {S,U}XTWhas no matching C function underneath.Checking with the team, it seems that they hit the same issue and decided that it's simpler to not have a specific API call and let the user separate it out to an extension and load call, which can then be optimised by the compiler:
https://godbolt.org/z/7K74h9Erf
tannergooding commentedon Feb 14, 2024
If there is a simple pattern we can use, then that works. I forgot that most of the SVE methods work in even/odd pairs so it should be natural to recognize
Gather(..., WidenEven(...))(or w/e the appropriate name is here).a74nh commentedon Apr 15, 2025
All of these are in Sve.cs
Available as experimental API in .NET 9.
a74nh commentedon May 2, 2025
Added #115241 to continue the discussion for adding offset as an argument.
a74nh commentedon May 2, 2025
As per the discussion on #104502 (comment)
The actual approved versions of the FFR methods are: