diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs index e1155f341de036..761c949bd5bfcb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/ASCIIUtility.cs @@ -662,7 +662,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin Debug.Assert(Sse2.IsSupported, "Should've been checked by caller."); Debug.Assert(BitConverter.IsLittleEndian, "SSE2 assumes little-endian."); - Vector128 firstVector, secondVector; + Vector128 firstVector, secondVector; uint currentMask; char* pOriginalBuffer = pBuffer; @@ -675,33 +675,23 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin // jumps as much as possible in the optimistic case of "all ASCII". If we see non-ASCII // data, we jump out of the hot paths to targets at the end of the method. - Vector128 asciiMaskForPTEST = Vector128.Create(unchecked((short)0xFF80)); // used for PTEST on supported hardware - Vector128 asciiMaskForPMINUW = Vector128.Create((ushort)0x0080); // used for PMINUW on supported hardware - Vector128 asciiMaskForPXOR = Vector128.Create(unchecked((short)0x8000)); // used for PXOR - Vector128 asciiMaskForPCMPGTW = Vector128.Create(unchecked((short)0x807F)); // used for PCMPGTW + Vector128 asciiMaskForPTEST = Vector128.Create((ushort)0xFF80); // used for PTEST on supported hardware + Vector128 asciiMaskForPADDUSW = Vector128.Create((ushort)0x7F80); // used for PADDUSW + const uint NonAsciiDataSeenMask = 0b_1010_1010_1010_1010; // used for determining whether 'currentMask' contains non-ASCII data Debug.Assert(bufferLength <= nuint.MaxValue / sizeof(char)); // Read the first vector unaligned. - firstVector = Sse2.LoadVector128((short*)pBuffer); // unaligned load + firstVector = Sse2.LoadVector128((ushort*)pBuffer); // unaligned load - if (Sse41.IsSupported) - { - // The SSE41-optimized code path works by forcing the 0x0080 bit in each WORD of the vector to be - // set iff the WORD element has value >= 0x0080 (non-ASCII). Then we'll treat it as a BYTE vector - // in order to extract the mask. - currentMask = (uint)Sse2.MoveMask(Sse41.Min(firstVector.AsUInt16(), asciiMaskForPMINUW).AsByte()); - } - else - { - // The SSE2-optimized code path works by forcing each WORD of the vector to be 0xFFFF iff the WORD - // element has value >= 0x0080 (non-ASCII). Then we'll treat it as a BYTE vector in order to extract - // the mask. - currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()); - } + // The operation below forces the 0x8000 bit of each WORD to be set iff the WORD element + // has value >= 0x0800 (non-ASCII). Then we'll treat the vector as a BYTE vector in order + // to extract the mask. Reminder: the 0x0080 bit of each WORD should be ignored. - if (currentMask != 0) + currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte()); + + if ((currentMask & NonAsciiDataSeenMask) != 0) { goto FoundNonAsciiDataInCurrentMask; } @@ -745,9 +735,9 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin do { - firstVector = Sse2.LoadAlignedVector128((short*)pBuffer); - secondVector = Sse2.LoadAlignedVector128((short*)pBuffer + SizeOfVector128InChars); - Vector128 combinedVector = Sse2.Or(firstVector, secondVector); + firstVector = Sse2.LoadAlignedVector128((ushort*)pBuffer); + secondVector = Sse2.LoadAlignedVector128((ushort*)pBuffer + SizeOfVector128InChars); + Vector128 combinedVector = Sse2.Or(firstVector, secondVector); if (Sse41.IsSupported) { @@ -761,7 +751,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin else { // See comment earlier in the method for an explanation of how the below logic works. - if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(combinedVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0) + currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(combinedVector, asciiMaskForPADDUSW).AsByte()); + if ((currentMask & NonAsciiDataSeenMask) != 0) { goto FoundNonAsciiDataInFirstOrSecondVector; } @@ -789,7 +780,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin // At least one full vector's worth of data remains, so we can safely read it. // Remember, at this point pBuffer is still aligned. - firstVector = Sse2.LoadAlignedVector128((short*)pBuffer); + firstVector = Sse2.LoadAlignedVector128((ushort*)pBuffer); if (Sse41.IsSupported) { @@ -803,8 +794,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin else { // See comment earlier in the method for an explanation of how the below logic works. - currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()); - if (currentMask != 0) + currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte()); + if ((currentMask & NonAsciiDataSeenMask) != 0) { goto FoundNonAsciiDataInCurrentMask; } @@ -822,7 +813,7 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin // We need to adjust the pointer because we're re-reading data. pBuffer = (char*)((byte*)pBuffer + (bufferLength & (SizeOfVector128InBytes - 1)) - SizeOfVector128InBytes); - firstVector = Sse2.LoadVector128((short*)pBuffer); // unaligned load + firstVector = Sse2.LoadVector128((ushort*)pBuffer); // unaligned load if (Sse41.IsSupported) { @@ -836,8 +827,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin else { // See comment earlier in the method for an explanation of how the below logic works. - currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()); - if (currentMask != 0) + currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte()); + if ((currentMask & NonAsciiDataSeenMask) != 0) { goto FoundNonAsciiDataInCurrentMask; } @@ -867,8 +858,8 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin } else { - currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()); - if (currentMask != 0) + currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte()); + if ((currentMask & NonAsciiDataSeenMask) != 0) { goto FoundNonAsciiDataInCurrentMask; } @@ -882,24 +873,27 @@ private static unsafe nuint GetIndexOfFirstNonAsciiChar_Sse2(char* pBuffer, nuin FoundNonAsciiDataInFirstVector: // See comment earlier in the method for an explanation of how the below logic works. - if (Sse41.IsSupported) - { - currentMask = (uint)Sse2.MoveMask(Sse41.Min(firstVector.AsUInt16(), asciiMaskForPMINUW).AsByte()); - } - else - { - currentMask = (uint)Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(firstVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()); - } + currentMask = (uint)Sse2.MoveMask(Sse2.AddSaturate(firstVector, asciiMaskForPADDUSW).AsByte()); FoundNonAsciiDataInCurrentMask: - // The mask contains - from the LSB - a 0 for each ASCII byte we saw, and a 1 for each non-ASCII byte. - // Tzcnt is the correct operation to count the number of zero bits quickly. If this instruction isn't - // available, we'll fall back to a normal loop. (Even though the original vector used WORD elements, - // masks work on BYTE elements, and we account for this in the final fixup.) + // See comment earlier in the method accounting for the 0x8000 and 0x0080 bits set after the WORD-sized operations. + + currentMask &= NonAsciiDataSeenMask; + + // Now, the mask contains - from the LSB - a 0b00 pair for each ASCII char we saw, and a 0b10 pair for each non-ASCII char. + // + // (Keep endianness in mind in the below examples.) + // A non-ASCII char followed by two ASCII chars is 0b..._00_00_10. (tzcnt = 1) + // An ASCII char followed by two non-ASCII chars is 0b..._10_10_00. (tzcnt = 3) + // Two ASCII chars followed by a non-ASCII char is 0b..._10_00_00. (tzcnt = 5) + // + // This means tzcnt = 2 * numLeadingAsciiChars + 1. We can conveniently take advantage of the fact + // that the 2x multiplier already matches the char* stride length, then just subtract 1 at the end to + // compute the correct final ending pointer value. Debug.Assert(currentMask != 0, "Shouldn't be here unless we see non-ASCII data."); - pBuffer = (char*)((byte*)pBuffer + (uint)BitOperations.TrailingZeroCount(currentMask)); + pBuffer = (char*)((byte*)pBuffer + (uint)BitOperations.TrailingZeroCount(currentMask) - 1); goto Finish; @@ -1315,8 +1309,8 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA Debug.Assert(elementCount >= 2 * SizeOfVector128); Vector128 asciiMaskForPTEST = Vector128.Create(unchecked((short)0xFF80)); // used for PTEST on supported hardware - Vector128 asciiMaskForPXOR = Vector128.Create(unchecked((short)0x8000)); // used for PXOR - Vector128 asciiMaskForPCMPGTW = Vector128.Create(unchecked((short)0x807F)); // used for PCMPGTW + Vector128 asciiMaskForPADDUSW = Vector128.Create((ushort)0x7F80); // used for PADDUSW + const int NonAsciiDataSeenMask = 0b_1010_1010_1010_1010; // used for determining whether the pmovmskb operation saw non-ASCII chars // First, perform an unaligned read of the first part of the input buffer. @@ -1334,7 +1328,7 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA } else { - if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(utf16VectorFirst, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0) + if ((Sse2.MoveMask(Sse2.AddSaturate(utf16VectorFirst.AsUInt16(), asciiMaskForPADDUSW).AsByte()) & NonAsciiDataSeenMask) != 0) { return 0; } @@ -1374,7 +1368,7 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA } else { - if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(utf16VectorFirst, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0) + if ((Sse2.MoveMask(Sse2.AddSaturate(utf16VectorFirst.AsUInt16(), asciiMaskForPADDUSW).AsByte()) & NonAsciiDataSeenMask) != 0) { goto Finish; } @@ -1413,7 +1407,7 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA } else { - if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(combinedVector, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0) + if ((Sse2.MoveMask(Sse2.AddSaturate(combinedVector.AsUInt16(), asciiMaskForPADDUSW).AsByte()) & NonAsciiDataSeenMask) != 0) { goto FoundNonAsciiDataInLoop; } @@ -1447,7 +1441,7 @@ private static unsafe nuint NarrowUtf16ToAscii_Sse2(char* pUtf16Buffer, byte* pA } else { - if (Sse2.MoveMask(Sse2.CompareGreaterThan(Sse2.Xor(utf16VectorFirst, asciiMaskForPXOR), asciiMaskForPCMPGTW).AsByte()) != 0) + if ((Sse2.MoveMask(Sse2.AddSaturate(utf16VectorFirst.AsUInt16(), asciiMaskForPADDUSW).AsByte()) & NonAsciiDataSeenMask) != 0) { goto Finish; // found non-ASCII data } diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs index 7c05368062d8b2..be733006d599a5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf16Utility.Validation.cs @@ -88,48 +88,47 @@ static Utf16Utility() Vector128 utf16Data = Sse2.LoadVector128((ushort*)pInputBuffer); // unaligned uint mask; - // The 'charIsNonAscii' vector we're about to build will have the 0x8000 or the 0x0080 - // bit set (but not both!) only if the corresponding input char is non-ASCII. Which of - // the two bits is set doesn't matter, as will be explained in the diagram a few lines - // below. - Vector128 charIsNonAscii; if (Sse41.IsSupported) { - // sets 0x0080 bit if corresponding char element is >= 0x0080 + // Sets the 0x0080 bit of each element in 'charIsNonAscii' if the corresponding + // input was 0x0080 <= [value]. (i.e., [value] is non-ASCII.) + charIsNonAscii = Sse41.Min(utf16Data, vector0080); } else { - // sets 0x8000 bit if corresponding char element is >= 0x0080 - charIsNonAscii = Sse2.AndNot(vector0080, Sse2.Subtract(vectorZero, Sse2.ShiftRightLogical(utf16Data, 7))); + // Sets the 0x0080 bit of each element in 'charIsNonAscii' if the corresponding + // input was 0x0080 <= [value] <= 0x7FFF. The case where 0x8000 <= [value] will + // be handled in a few lines. + + charIsNonAscii = Sse2.AndNot(Sse2.CompareGreaterThan(vector0080.AsInt16(), utf16Data.AsInt16()).AsUInt16(), vector0080); } #if DEBUG - // Quick check to ensure we didn't accidentally set both 0x8080 bits in any element. + // Quick check to ensure we didn't accidentally set the 0x8000 bit of any element. uint debugMask = (uint)Sse2.MoveMask(charIsNonAscii.AsByte()); - Debug.Assert((debugMask & (debugMask << 1)) == 0, "Two set bits shouldn't occur adjacent to each other in this mask."); + Debug.Assert((debugMask & 0b_1010_1010_1010_1010) == 0, "Shouldn't have set the 0x8000 bit of any element in 'charIsNonAscii'."); #endif // DEBUG - // sets 0x8080 bits if corresponding char element is >= 0x0800 + // Sets the 0x8080 bits of each element in 'charIsNonAscii' if the corresponding + // input was 0x0800 <= [value]. This also handles the missing range a few lines above. + Vector128 charIsThreeByteUtf8Encoded = Sse2.Subtract(vectorZero, Sse2.ShiftRightLogical(utf16Data, 11)); mask = (uint)Sse2.MoveMask(Sse2.Or(charIsNonAscii, charIsThreeByteUtf8Encoded).AsByte()); - // Each odd bit of mask will be 1 only if the char was >= 0x0080, - // and each even bit of mask will be 1 only if the char was >= 0x0800. + // Each even bit of mask will be 1 only if the char was >= 0x0080, + // and each odd bit of mask will be 1 only if the char was >= 0x0800. // // Example for UTF-16 input "[ 0123 ] [ 1234 ] ...": // - // ,-- set if char[1] is non-ASCII - // | ,-- set if char[0] is non-ASCII + // ,-- set if char[1] is >= 0x0800 + // | ,-- set if char[0] is >= 0x0800 // v v - // mask = ... 1 1 1 0 - // ^ ^-- set if char[0] is >= 0x0800 - // `-- set if char[1] is >= 0x0800 - // - // (If the SSE4.1 code path is taken above, the meaning of the odd and even - // bits are swapped, but the logic below otherwise holds.) + // mask = ... 1 1 0 1 + // ^ ^-- set if char[0] is non-ASCII + // `-- set if char[1] is non-ASCII // // This means we can popcnt the number of set bits, and the result is the // number of *additional* UTF-8 bytes that each UTF-16 code unit requires as