Skip to content

Tighten up some SSE2 ASCII + UTF-16 transcoding logic #32036

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
GrabYourPitchforks merged 2 commits into from
Feb 11, 2020
Merged

Tighten up some SSE2 ASCII + UTF-16 transcoding logic #32036

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0f4acf9
Tighten up some SSE2 ASCII + UTF-16 transcoding logic
GrabYourPitchforks Jan 21, 2020
7da52ac
Avoid unnecessary sign extension after pmovmskb
GrabYourPitchforks Feb 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 46 additions & 52 deletions src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -662,7 +662,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin
Debug.Assert(Sse2.IsSupported, "Should've been checked by caller.");
Debug.Assert(BitConverter.IsLittleEndian, "SSE2 assumes little-endian.");

Vector128<short> firstVector, secondVector;
Vector128<ushort> firstVector, secondVector;
uint currentMask;
char* pOriginalBuffer = pBuffer;

Expand All @@ -675,33 +675,23 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin
// jumps as much as possible in the optimistic case of "all ASCII". If we see non-ASCII
// data, we jump out of the hot paths to targets at the end of the method.

Vector128<short> asciiMaskForPTEST = Vector128.Create(unchecked((short)0xFF80)); // used for PTEST on supported hardware
Vector128<ushort> asciiMaskForPMINUW = Vector128.Create((ushort)0x0080); // used for PMINUW on supported hardware
Vector128<short> asciiMaskForPXOR = Vector128.Create(unchecked((short)0x8000)); // used for PXOR
Vector128<short> asciiMaskForPCMPGTW = Vector128.Create(unchecked((short)0x807F)); // used for PCMPGTW
Vector128<ushort> asciiMaskForPTEST = Vector128.Create((ushort)0xFF80); // used for PTEST on supported hardware
Vector128<ushort> asciiMaskForPADDUSW = Vector128.Create((ushort)0x7F80); // used for PADDUSW
const uint NonAsciiDataSeenMask = 0b_1010_1010_1010_1010; // used for determining whether 'currentMask' contains non-ASCII data

Debug.Assert(bufferLength <= nuint.MaxValue / sizeof(char));

// Read the first vector unaligned.

firstVector = Sse2.LoadVector128((short*)pBuffer); // unaligned load
firstVector = Sse2.LoadVector128((ushort*)pBuffer); // unaligned load

if (Sse41.IsSupported)
{
// The SSE41-optimized code path works by forcing the 0x0080 bit in each WORD of the vector to be
// set iff the WORD element has value >= 0x0080 (non-ASCII). Then we'll treat it as a BYTE vector
// in order to extract the mask.
currentMask = (uint)Sse2.MoveMask(Sse41.Min(firstVector.AsUInt16(), asciiMaskForPMINUW).AsByte());
}
else
{
// The SSE2-optimized code path works by forcing each WORD of the vector to be 0xFFFF iff the WORD
// element has value >= 0x0080 (non-ASCII). Then we'll treat it as a BYTE vector in order to extract
// the mask.
currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte());
}
// The operation below forces the 0x8000 bit of each WORD to be set iff the WORD element
// has value >= 0x0800 (non-ASCII). Then we'll treat the vector as a BYTE vector in order
// to extract the mask. Reminder: the 0x0080 bit of each WORD should be ignored.

if (currentMask != 0)
currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte());

if ((currentMask & NonAsciiDataSeenMask) != 0)
{
goto FoundNonAsciiDataInCurrentMask;
}
Expand Down Expand Up @@ -745,9 +735,9 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin

do
{
firstVector = Sse2.LoadAlignedVector128((short*)pBuffer);
secondVector = Sse2.LoadAlignedVector128((short*)pBuffer + SizeOfVector128InChars);
Vector128<short> combinedVector = Sse2.Or(firstVector, secondVector);
firstVector = Sse2.LoadAlignedVector128((ushort*)pBuffer);
secondVector = Sse2.LoadAlignedVector128((ushort*)pBuffer + SizeOfVector128InChars);
Vector128<ushort> combinedVector = Sse2.Or(firstVector, secondVector);

if (Sse41.IsSupported)
{
Expand All @@ -761,7 +751,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin
else
{
// See comment earlier in the method for an explanation of how the below logic works.
if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(combinedVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0)
currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(combinedVector, asciiMaskForPADDUSW).AsByte());
if ((currentMask & NonAsciiDataSeenMask) != 0)
{
goto FoundNonAsciiDataInFirstOrSecondVector;
}
Expand Down Expand Up @@ -789,7 +780,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin
// At least one full vector's worth of data remains, so we can safely read it.
// Remember, at this point pBuffer is still aligned.

firstVector = Sse2.LoadAlignedVector128((short*)pBuffer);
firstVector = Sse2.LoadAlignedVector128((ushort*)pBuffer);

if (Sse41.IsSupported)
{
Expand All @@ -803,8 +794,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin
else
{
// See comment earlier in the method for an explanation of how the below logic works.
currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte());
if (currentMask != 0)
currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte());
if ((currentMask & NonAsciiDataSeenMask) != 0)
{
goto FoundNonAsciiDataInCurrentMask;
}
Expand All @@ -822,7 +813,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin
// We need to adjust the pointer because we're re-reading data.

pBuffer = (char*)((byte*)pBuffer + (bufferLength & (SizeOfVector128InBytes - 1)) - SizeOfVector128InBytes);
firstVector = Sse2.LoadVector128((short*)pBuffer); // unaligned load
firstVector = Sse2.LoadVector128((ushort*)pBuffer); // unaligned load

if (Sse41.IsSupported)
{
Expand All @@ -836,8 +827,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin
else
{
// See comment earlier in the method for an explanation of how the below logic works.
currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte());
if (currentMask != 0)
currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte());
if ((currentMask & NonAsciiDataSeenMask) != 0)
{
goto FoundNonAsciiDataInCurrentMask;
}
Expand Down Expand Up @@ -867,8 +858,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin
}
else
{
currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte());
if (currentMask != 0)
currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte());
if ((currentMask & NonAsciiDataSeenMask) != 0)
{
goto FoundNonAsciiDataInCurrentMask;
}
Expand All @@ -882,24 +873,27 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin
FoundNonAsciiDataInFirstVector:

// See comment earlier in the method for an explanation of how the below logic works.
if (Sse41.IsSupported)
{
currentMask = (uint)Sse2.MoveMask(Sse41.Min(firstVector.AsUInt16(), asciiMaskForPMINUW).AsByte());
}
else
{
currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte());
}
currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte());

FoundNonAsciiDataInCurrentMask:

// The mask contains - from the LSB - a 0 for each ASCII byte we saw, and a 1 for each non-ASCII byte.
// Tzcnt is the correct operation to count the number of zero bits quickly. If this instruction isn't
// available, we'll fall back to a normal loop. (Even though the original vector used WORD elements,
// masks work on BYTE elements, and we account for this in the final fixup.)
// See comment earlier in the method accounting for the 0x8000 and 0x0080 bits set after the WORD-sized operations.

currentMask &= NonAsciiDataSeenMask;

// Now, the mask contains - from the LSB - a 0b00 pair for each ASCII char we saw, and a 0b10 pair for each non-ASCII char.
//
// (Keep endianness in mind in the below examples.)
// A non-ASCII char followed by two ASCII chars is 0b..._00_00_10. (tzcnt = 1)
// An ASCII char followed by two non-ASCII chars is 0b..._10_10_00. (tzcnt = 3)
// Two ASCII chars followed by a non-ASCII char is 0b..._10_00_00. (tzcnt = 5)
//
// This means tzcnt = 2 * numLeadingAsciiChars + 1. We can conveniently take advantage of the fact
// that the 2x multiplier already matches the char* stride length, then just subtract 1 at the end to
// compute the correct final ending pointer value.

Debug.Assert(currentMask != 0, "Shouldn't be here unless we see non-ASCII data.");
pBuffer = (char*)((byte*)pBuffer + (uint)BitOperations.TrailingZeroCount(currentMask));
pBuffer = (char*)((byte*)pBuffer + (uint)BitOperations.TrailingZeroCount(currentMask) - 1);

goto Finish;

Expand Down Expand Up @@ -1315,8 +1309,8 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA
Debug.Assert(elementCount >= 2 * SizeOfVector128);

Vector128<short> asciiMaskForPTEST = Vector128.Create(unchecked((short)0xFF80)); // used for PTEST on supported hardware
Vector128<short> asciiMaskForPXOR = Vector128.Create(unchecked((short)0x8000)); // used for PXOR
Vector128<short> asciiMaskForPCMPGTW = Vector128.Create(unchecked((short)0x807F)); // used for PCMPGTW
Vector128<ushort> asciiMaskForPADDUSW = Vector128.Create((ushort)0x7F80); // used for PADDUSW
const int NonAsciiDataSeenMask = 0b_1010_1010_1010_1010; // used for determining whether the pmovmskb operation saw non-ASCII chars

// First, perform an unaligned read of the first part of the input buffer.

Expand All @@ -1334,7 +1328,7 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA
}
else
{
if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(utf16VectorFirst, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0)
if ((Sse2.MoveMask(Sse2.AddSaturate(utf16VectorFirst.AsUInt16(), asciiMaskForPADDUSW).AsByte()) & NonAsciiDataSeenMask) != 0)
{
return 0;
}
Expand Down Expand Up @@ -1374,7 +1368,7 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA
}
else
{
if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(utf16VectorFirst, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0)
if ((Sse2.MoveMask(Sse2.AddSaturate(utf16VectorFirst.AsUInt16(), asciiMaskForPADDUSW).AsByte()) & NonAsciiDataSeenMask) != 0)
{
goto Finish;
}
Expand Down Expand Up @@ -1413,7 +1407,7 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA
}
else
{
if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(combinedVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0)
if ((Sse2.MoveMask(Sse2.AddSaturate(combinedVector.AsUInt16(), asciiMaskForPADDUSW).AsByte()) & NonAsciiDataSeenMask) != 0)
{
goto FoundNonAsciiDataInLoop;
}
Expand Down Expand Up @@ -1447,7 +1441,7 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA
}
else
{
if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(utf16VectorFirst, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0)
if ((Sse2.MoveMask(Sse2.AddSaturate(utf16VectorFirst.AsUInt16(), asciiMaskForPADDUSW).AsByte()) & NonAsciiDataSeenMask) != 0)
{
goto Finish; // found non-ASCII data
}
Expand Down
41 changes: 20 additions & 21 deletions src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,48 +88,47 @@ static Utf16Utility()
Vector128<ushort> utf16Data = Sse2.LoadVector128((ushort*)pInputBuffer); // unaligned
uint mask;

// The 'charIsNonAscii' vector we're about to build will have the 0x8000 or the 0x0080
// bit set (but not both!) only if the corresponding input char is non-ASCII. Which of
// the two bits is set doesn't matter, as will be explained in the diagram a few lines
// below.

Vector128<ushort> charIsNonAscii;
if (Sse41.IsSupported)
{
// sets 0x0080 bit if corresponding char element is >= 0x0080
// Sets the 0x0080 bit of each element in 'charIsNonAscii' if the corresponding
// input was 0x0080 <= [value]. (i.e., [value] is non-ASCII.)

charIsNonAscii = Sse41.Min(utf16Data, vector0080);
}
else
{
// sets 0x8000 bit if corresponding char element is >= 0x0080
charIsNonAscii = Sse2.AndNot(vector0080, Sse2.Subtract(vectorZero, Sse2.ShiftRightLogical(utf16Data, 7)));
// Sets the 0x0080 bit of each element in 'charIsNonAscii' if the corresponding
// input was 0x0080 <= [value] <= 0x7FFF. The case where 0x8000 <= [value] will
// be handled in a few lines.

charIsNonAscii = Sse2.AndNot(Sse2.CompareGreaterThan(vector0080.AsInt16(), utf16Data.AsInt16()).AsUInt16(), vector0080);
}

#if DEBUG
// Quick check to ensure we didn't accidentally set both 0x8080 bits in any element.
// Quick check to ensure we didn't accidentally set the 0x8000 bit of any element.
uint debugMask = (uint)Sse2.MoveMask(charIsNonAscii.AsByte());
Debug.Assert((debugMask & (debugMask << 1)) == 0, "Two set bits shouldn't occur adjacent to each other in this mask.");
Debug.Assert((debugMask & 0b_1010_1010_1010_1010) == 0, "Shouldn't have set the 0x8000 bit of any element in 'charIsNonAscii'.");
#endif // DEBUG

// sets 0x8080 bits if corresponding char element is >= 0x0800
// Sets the 0x8080 bits of each element in 'charIsNonAscii' if the corresponding
// input was 0x0800 <= [value]. This also handles the missing range a few lines above.

Vector128<ushort> charIsThreeByteUtf8Encoded = Sse2.Subtract(vectorZero, Sse2.ShiftRightLogical(utf16Data, 11));

mask = (uint)Sse2.MoveMask(Sse2.Or(charIsNonAscii, charIsThreeByteUtf8Encoded).AsByte());

// Each odd bit of mask will be 1 only if the char was >= 0x0080,
// and each even bit of mask will be 1 only if the char was >= 0x0800.
// Each even bit of mask will be 1 only if the char was >= 0x0080,
// and each odd bit of mask will be 1 only if the char was >= 0x0800.
//
// Example for UTF-16 input "[ 0123 ] [ 1234 ] ...":
//
// ,-- set if char[1] is non-ASCII
// | ,-- set if char[0] is non-ASCII
// ,-- set if char[1] is >= 0x0800
// | ,-- set if char[0] is >= 0x0800
// v v
// mask = ... 1 1 1 0
// ^ ^-- set if char[0] is >= 0x0800
// `-- set if char[1] is >= 0x0800
//
// (If the SSE4.1 code path is taken above, the meaning of the odd and even
// bits are swapped, but the logic below otherwise holds.)
// mask = ... 1 1 0 1
// ^ ^-- set if char[0] is non-ASCII
// `-- set if char[1] is non-ASCII
//
// This means we can popcnt the number of set bits, and the result is the
// number of *additional* UTF-8 bytes that each UTF-16 code unit requires as
Expand Down