diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 4b3d13059e977c..adec712b4e0074 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -822,38 +822,75 @@ void CodeGen::genCodeForMulHi(GenTreeOp* treeNode) // to get the high bits of the multiply, we are constrained to using the // 1-op form: RDX:RAX = RAX * rm // The 3-op form (Rx=Ry*Rz) does not support it. - + // When BMI2 is available, we can use the MULX instruction to get the high bits genConsumeOperands(treeNode->AsOp()); GenTree* regOp = op1; GenTree* rmOp = op2; - // Set rmOp to the memory operand (if any) - if (op1->isUsedFromMemory() || (op2->isUsedFromReg() && (op2->GetRegNum() == REG_RAX))) + if (op1->isUsedFromMemory()) { regOp = op2; rmOp = op1; } assert(regOp->isUsedFromReg()); - // Setup targetReg when neither of the source operands was a matching register - inst_Mov(targetType, REG_RAX, regOp->GetRegNum(), /* canSkip */ true); - - instruction ins; - if ((treeNode->gtFlags & GTF_UNSIGNED) == 0) + if (treeNode->IsUnsigned() && compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) { - ins = INS_imulEAX; + if (rmOp->isUsedFromReg() && (rmOp->GetRegNum() == REG_RDX)) + { + std::swap(regOp, rmOp); + } + + // Setup targetReg when neither of the source operands was a matching register + inst_Mov(targetType, REG_RDX, regOp->GetRegNum(), /* canSkip */ true); + + if (treeNode->OperIs(GT_MULHI)) + { + // emit MULX instruction, use targetReg twice to only store high result + inst_RV_RV_TT(INS_mulx, size, targetReg, targetReg, rmOp, /* isRMW */ false, INS_OPTS_NONE); + } + else + { +#if TARGET_64BIT + assert(false); +#else + assert(treeNode->OperIs(GT_MUL_LONG)); + + // emit MULX instruction + regNumber hiReg = treeNode->AsMultiRegOp()->GetRegByIndex(1); + inst_RV_RV_TT(INS_mulx, size, hiReg, targetReg, rmOp, /* isRMW */ false, INS_OPTS_NONE); +#endif + } } - else + else // Generate MUL or IMUL instruction { - ins = INS_mulEAX; - } - emit->emitInsBinary(ins, size, treeNode, rmOp); + // If op2 is already present in RAX use that as implicit operand + if (rmOp->isUsedFromReg() && (rmOp->GetRegNum() == REG_RAX)) + { + std::swap(regOp, rmOp); + } - // Move the result to the desired register, if necessary - if (treeNode->OperIs(GT_MULHI)) - { - inst_Mov(targetType, targetReg, REG_RDX, /* canSkip */ true); + // Setup targetReg when neither of the source operands was a matching register + inst_Mov(targetType, REG_RAX, regOp->GetRegNum(), /* canSkip */ true); + + instruction ins; + if (!treeNode->IsUnsigned()) + { + ins = INS_imulEAX; + } + else + { + ins = INS_mulEAX; + } + emit->emitInsBinary(ins, size, treeNode, rmOp); + + // Move the result to the desired register, if necessary + if (treeNode->OperIs(GT_MULHI)) + { + assert(targetReg == REG_RDX); + inst_Mov(targetType, targetReg, REG_RDX, /* canSkip */ true); + } } genProduceReg(treeNode); diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index d27dec517ada88..2d357a54c78b96 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -7934,11 +7934,12 @@ bool Lowering::LowerUnsignedDivOrMod(GenTreeOp* divMod) } #ifdef TARGET_XARCH - // force input transformation to RAX because the following MULHI will kill RDX:RAX anyway and LSRA often causes - // redundant copies otherwise + // force input transformation to RAX/RDX because the following MULHI will kill RDX:RAX (RDX if mulx is + // available) anyway and LSRA often causes redundant copies otherwise if (firstNode && !simpleMul) { - adjustedDividend->SetRegNum(REG_RAX); + regNumber implicitReg = comp->compOpportunisticallyDependsOn(InstructionSet_AVX2) ? REG_RDX : REG_RAX; + adjustedDividend->SetRegNum(implicitReg); } #endif diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 757bafcc7a9f97..8add7cdc88b95c 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -496,6 +496,13 @@ class Lowering final : public Phase #endif // TARGET_XARCH +#if TARGET_X86 + if (parentNode->OperIs(GT_MUL_LONG)) + { + return genTypeSize(childNode->TypeGet()) == operatorSize / 2; + } +#endif // TARGET_X86 + return genTypeSize(childNode->TypeGet()) == operatorSize; } diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index d703239f894d08..7f7424214dc93d 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -7851,7 +7851,7 @@ void Lowering::ContainCheckMul(GenTreeOp* node) bool isSafeToContainOp1 = true; bool isSafeToContainOp2 = true; - bool isUnsignedMultiply = ((node->gtFlags & GTF_UNSIGNED) != 0); + bool isUnsignedMultiply = node->IsUnsigned(); bool requiresOverflowCheck = node->gtOverflowEx(); bool useLeaEncoding = false; GenTree* memOp = nullptr; @@ -7859,6 +7859,7 @@ void Lowering::ContainCheckMul(GenTreeOp* node) bool hasImpliedFirstOperand = false; GenTreeIntConCommon* imm = nullptr; GenTree* other = nullptr; + var_types nodeType = node->TypeGet(); // Multiply should never be using small types assert(!varTypeIsSmall(node->TypeGet())); @@ -7878,6 +7879,8 @@ void Lowering::ContainCheckMul(GenTreeOp* node) else if (node->OperIs(GT_MUL_LONG)) { hasImpliedFirstOperand = true; + // GT_MUL_LONG hsa node type LONG but work on INT + nodeType = TYP_INT; } #endif else if (IsContainableImmed(node, op2) || IsContainableImmed(node, op1)) @@ -7914,7 +7917,7 @@ void Lowering::ContainCheckMul(GenTreeOp* node) // if (memOp == nullptr) { - if ((op2->TypeGet() == node->TypeGet()) && IsContainableMemoryOp(op2)) + if ((op2->TypeGet() == nodeType) && IsContainableMemoryOp(op2)) { isSafeToContainOp2 = IsSafeToContainMem(node, op2); if (isSafeToContainOp2) @@ -7923,7 +7926,7 @@ void Lowering::ContainCheckMul(GenTreeOp* node) } } - if ((memOp == nullptr) && (op1->TypeGet() == node->TypeGet()) && IsContainableMemoryOp(op1)) + if ((memOp == nullptr) && (op1->TypeGet() == nodeType) && IsContainableMemoryOp(op1)) { isSafeToContainOp1 = IsSafeToContainMem(node, op1); if (isSafeToContainOp1) @@ -7934,7 +7937,7 @@ void Lowering::ContainCheckMul(GenTreeOp* node) } else { - if ((memOp->TypeGet() != node->TypeGet())) + if ((memOp->TypeGet() != nodeType)) { memOp = nullptr; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 0d297d92a929e5..4a618a39c53fb8 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -782,7 +782,27 @@ regMaskTP LinearScan::getKillSetForMul(GenTreeOp* mulNode) regMaskTP killMask = RBM_NONE; #ifdef TARGET_XARCH assert(mulNode->OperIsMul()); - if (!mulNode->OperIs(GT_MUL) || (((mulNode->gtFlags & GTF_UNSIGNED) != 0) && mulNode->gtOverflowEx())) + if (!mulNode->OperIs(GT_MUL)) + { + // If we can use the mulx instruction, we don't need to kill RAX + if (mulNode->IsUnsigned() && compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + // If on operand is contained, we define fixed RDX register for use, so we don't need to kill register. + if (mulNode->gtGetOp1()->isContained() || mulNode->gtGetOp2()->isContained()) + { + killMask = RBM_NONE; + } + else + { + killMask = RBM_RDX; + } + } + else + { + killMask = RBM_RAX | RBM_RDX; + } + } + else if (mulNode->IsUnsigned() && mulNode->gtOverflowEx()) { killMask = RBM_RAX | RBM_RDX; } diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 00d8d8c40cc5bf..3f38829606af50 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -795,6 +795,14 @@ bool LinearScan::isRMWRegOper(GenTree* tree) } return (!tree->gtGetOp2()->isContainedIntOrIImmed() && !tree->gtGetOp1()->isContainedIntOrIImmed()); } +#ifdef TARGET_X86 + case GT_MUL_LONG: +#endif + case GT_MULHI: + { + // MUL, IMUL are RMW but mulx is not (which is used for unsigned operands when BMI2 is availible) + return !(tree->IsUnsigned() && compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)); + } #ifdef FEATURE_HW_INTRINSICS case GT_HWINTRINSIC: @@ -3223,18 +3231,22 @@ int LinearScan::BuildMul(GenTree* tree) return BuildSimple(tree); } - // ToDo-APX : imul currently doesn't have rex2 support. So, cannot use R16-R31. - int srcCount = BuildBinaryUses(tree->AsOp()); + bool isUnsignedMultiply = tree->IsUnsigned(); + bool requiresOverflowCheck = tree->gtOverflowEx(); + bool useMulx = + !tree->OperIs(GT_MUL) && isUnsignedMultiply && compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2); + + // ToDo-APX : imul currently doesn't have rex2 support. So, cannot use R16-R31. + int srcCount = 0; int dstCount = 1; SingleTypeRegSet dstCandidates = RBM_NONE; - bool isUnsignedMultiply = ((tree->gtFlags & GTF_UNSIGNED) != 0); - bool requiresOverflowCheck = tree->gtOverflowEx(); - - // There are three forms of x86 multiply: + // There are three forms of x86 multiply in base instruction set // one-op form: RDX:RAX = RAX * r/m // two-op form: reg *= r/m // three-op form: reg = r/m * imm + // If the BMI2 instruction set is supported there is an additional unsigned multiply + // mulx reg1:reg2 = RDX * reg3/m // This special widening 32x32->64 MUL is not used on x64 #if defined(TARGET_X86) @@ -3244,42 +3256,64 @@ int LinearScan::BuildMul(GenTree* tree) assert((tree->gtFlags & GTF_MUL_64RSLT) == 0); } - // We do use the widening multiply to implement - // the overflow checking for unsigned multiply - // - if (isUnsignedMultiply && requiresOverflowCheck) + if (useMulx) { - // The only encoding provided is RDX:RAX = RAX * rm - // - // Here we set RAX as the only destination candidate - // In LSRA we set the kill set for this operation to RBM_RAX|RBM_RDX - // - dstCandidates = SRBM_RAX; - } - else if (tree->OperIs(GT_MULHI)) - { - // Have to use the encoding:RDX:RAX = RAX * rm. Since we only care about the - // upper 32 bits of the result set the destination candidate to REG_RDX. - dstCandidates = SRBM_RDX; - } + // If one of the operands is contained, specify RDX for the other operand + SingleTypeRegSet srcCandidates1 = RBM_NONE; + SingleTypeRegSet srcCandidates2 = RBM_NONE; + if (op1->isContained()) + { + assert(!op2->isContained()); + srcCandidates2 = SRBM_RDX; + } + else if (op2->isContained()) + { + srcCandidates1 = SRBM_RDX; + } + + srcCount = BuildOperandUses(op1, srcCandidates1); + srcCount += BuildOperandUses(op2, srcCandidates2); + #if defined(TARGET_X86) - else if (tree->OperIs(GT_MUL_LONG)) - { - // have to use the encoding:RDX:RAX = RAX * rm - dstCandidates = SRBM_RAX | SRBM_RDX; - dstCount = 2; - } + if (tree->OperIs(GT_MUL_LONG)) + { + dstCount = 2; + } #endif - GenTree* containedMemOp = nullptr; - if (op1->isContained() && !op1->IsCnsIntOrI()) - { - assert(!op2->isContained() || op2->IsCnsIntOrI()); - containedMemOp = op1; } - else if (op2->isContained() && !op2->IsCnsIntOrI()) + else { - containedMemOp = op2; + assert(!(op1->isContained() && !op1->IsCnsIntOrI()) || !(op2->isContained() && !op2->IsCnsIntOrI())); + srcCount = BuildBinaryUses(tree->AsOp()); + + // We do use the widening multiply to implement + // the overflow checking for unsigned multiply + // + if (isUnsignedMultiply && requiresOverflowCheck) + { + // The only encoding provided is RDX:RAX = RAX * rm + // + // Here we set RAX as the only destination candidate + // In LSRA we set the kill set for this operation to RBM_RAX|RBM_RDX + // + dstCandidates = SRBM_RAX; + } + else if (tree->OperIs(GT_MULHI)) + { + // Have to use the encoding:RDX:RAX = RAX * rm. Since we only care about the + // upper 32 bits of the result set the destination candidate to REG_RDX. + dstCandidates = SRBM_RDX; + } +#if defined(TARGET_X86) + else if (tree->OperIs(GT_MUL_LONG)) + { + // We have to use the encoding:RDX:RAX = RAX * rm + dstCandidates = SRBM_RAX | SRBM_RDX; + dstCount = 2; + } +#endif } + regMaskTP killMask = getKillSetForMul(tree->AsOp()); BuildDefWithKills(tree, dstCount, dstCandidates, killMask); return srcCount;