[llvm] [IR] Add llvm `clmul` intrinsic (PR #140301)

Oscar Smith via llvm-commits llvm-commits at lists.llvm.org
Wed May 21 14:31:31 PDT 2025


https://github.com/oscardssmith updated https://github.com/llvm/llvm-project/pull/140301

>From c74bfec5b0224309a203d7c9636c8ff2b2c4318a Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscardssmith at gmail.com>
Date: Fri, 16 May 2025 12:15:08 -0400
Subject: [PATCH 1/8] add clmul docs

---
 llvm/docs/LangRef.rst                         | 48 +++++++++++++++++++
 llvm/include/llvm/CodeGen/ISDOpcodes.h        |  3 ++
 llvm/include/llvm/IR/Intrinsics.td            |  8 ++++
 llvm/lib/CodeGen/IntrinsicLowering.cpp        | 23 +++++++++
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  6 +++
 .../SelectionDAG/SelectionDAGDumper.cpp       |  1 +
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  2 +
 7 files changed, 91 insertions(+)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 343ca743c74f8..1444654a248e2 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -18072,6 +18072,54 @@ Example:
       %r = call i8 @llvm.fshr.i8(i8 15, i8 15, i8 11)  ; %r = i8: 225 (0b11100001)
       %r = call i8 @llvm.fshr.i8(i8 0, i8 255, i8 8)   ; %r = i8: 255 (0b11111111)
 
+.. clmul:
+
+'``clmul.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax
+"""""""
+
+This is an overloaded intrinsic. You can use ``llvm.clmul``
+on any integer bit width or vectors of integers.
+
+::
+
+      declare i16 @llvm.clmul.i16(i16 %a, i16 %b)
+      declare i32 @llvm.clmul.i32(i32 %a, i32 %b)
+      declare i64 @llvm.clmul.i64(i64 %a, i64 %b)
+      declare <4 x i32> @llvm.clmul.v4i32(<4 x i32> %a, <4 x i32> %b)
+
+Overview
+"""""""""
+
+The '``llvm.clmul``' family of intrinsics functions perform carryless multiplication
+(also known as xor multiplication) on the 2 arguments.
+
+Arguments
+""""""""""
+
+The arguments (%a and %b) and the result may be of integer types of any bit
+width, but they must have the same bit width. ``%a`` and ``%b`` are the two
+values that will undergo carryless multiplication.
+
+Semantics:
+""""""""""
+
+The ‘llvm.clmul’ intrinsic computes carryless multiply of ``%a`` and ``%b``, which is the result
+of applying the standard multiplication algorithm if you replace all of the aditions with exclusive ors.
+The vector intrinsics, such as llvm.clmul.v4i32, operate on a per-element basis and the element order is not affected.
+
+Examples
+"""""""""
+
+.. code-block:: llvm
+
+      %res = call i4 @llvm.clmul.i4(i4 1, i4 2)  ; %res = 2
+      %res = call i4 @llvm.clmul.i4(i4 5, i4 6)  ; %res = 14
+      %res = call i4 @llvm.clmul.i4(i4 -4, i4 2)  ; %res = -8
+      %res = call i4 @llvm.clmul.i4(i4 -4, i4 -5)  ; %res = -12
+
 Arithmetic with Overflow Intrinsics
 -----------------------------------
 
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 9f66402e4c820..fc3b3b26cbe5e 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -751,6 +751,9 @@ enum NodeType {
   ROTR,
   FSHL,
   FSHR,
+  
+  /// Carryless multiplication operator
+  CLMUL,
 
   /// Byte Swap and Counting operators.
   BSWAP,
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index e1a135a5ad48e..1857829910340 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1431,6 +1431,8 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in {
       [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
   def int_fshr : DefaultAttrsIntrinsic<[llvm_anyint_ty],
       [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
+  def int_clmul : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+      [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
 }
 
 let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn,
@@ -2103,6 +2105,12 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
                                LLVMMatchType<0>,
                                LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                llvm_i32_ty]>;
+  def int_vp_clmul : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
+                             [ LLVMMatchType<0>,
+                               LLVMMatchType<0>,
+                               LLVMMatchType<0>,
+                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                               llvm_i32_ty]>;
   def int_vp_sadd_sat : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
                              [ LLVMMatchType<0>,
                                LLVMMatchType<0>,
diff --git a/llvm/lib/CodeGen/IntrinsicLowering.cpp b/llvm/lib/CodeGen/IntrinsicLowering.cpp
index 1518ead7698be..d66544ee87ea4 100644
--- a/llvm/lib/CodeGen/IntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/IntrinsicLowering.cpp
@@ -199,6 +199,25 @@ static Value *LowerCTLZ(LLVMContext &Context, Value *V, Instruction *IP) {
   return LowerCTPOP(Context, V, IP);
 }
 
+/// Emit the code to lower clmul of V1, V2 before the specified instruction IP.
+static Value *LowerCLMUL(LLVMContext &Context, Value *V1, Value *V2, Instruction *IP) {
+
+  IRBuilder<> Builder(IP);
+
+  unsigned BitSize = V1->getType()->getPrimitiveSizeInBits();
+  Value *Res = ConstantInt::get(V1->getType(), 0);
+  Value *Zero = ConstantInt::get(V1->getType(), 0);
+  Value *One = ConstantInt::get(V1->getType(), 1);
+  for (unsigned I = 1; I < BitSize; I ++) {
+    Value *LowBit = Builder.CreateAnd(V1, One, "clmul.isodd");
+    Value *Pred = Builder.CreateSelect(LowBit, V2, Zero, "clmul.V2_or_zero");
+    Res = Builder.CreateXor(Res, Pred, "clmul.Res");
+    V1 = Builder.CreateLShr(V1, One, "clmul.V1");
+    V2 = Builder.CreateShl(V2, One, "clmul.V2");
+  }
+  return LowerCTPOP(Context, Res, IP);
+}
+
 static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname,
                                        const char *Dname,
                                        const char *LDname) {
@@ -262,6 +281,10 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
     CI->replaceAllUsesWith(LowerCTLZ(Context, CI->getArgOperand(0), CI));
     break;
 
+  case Intrinsic::clmul:
+    CI->replaceAllUsesWith(LowerCLMUL(Context, CI->getArgOperand(0), CI->getArgOperand(1), CI));
+    break;
+
   case Intrinsic::cttz: {
     // cttz(x) -> ctpop(~X & (X-1))
     Value *Src = CI->getArgOperand(0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 3ebd3a4b88097..a4e350702ebc8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7188,6 +7188,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     }
     return;
   }
+  case Intrinsic::clmul: {
+    SDValue Op1 = getValue(I.getArgOperand(0));
+    SDValue Op2 = getValue(I.getArgOperand(1));
+    setValue(&I, DAG.getNode(ISD::CLMUL, sdl, Op1.getValueType(), Op1, Op2));
+    return;
+  }
   case Intrinsic::sadd_sat: {
     SDValue Op1 = getValue(I.getArgOperand(0));
     SDValue Op2 = getValue(I.getArgOperand(1));
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 803894e298dd5..4fba332f806fd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -298,6 +298,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::ROTR:                       return "rotr";
   case ISD::FSHL:                       return "fshl";
   case ISD::FSHR:                       return "fshr";
+  case ISD::CLMUL:                      return "clmul";
   case ISD::FADD:                       return "fadd";
   case ISD::STRICT_FADD:                return "strict_fadd";
   case ISD::FSUB:                       return "fsub";
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5e761fccc815a..470685b9c68bb 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -10347,6 +10347,7 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     return DAG.getNode(RISCVISD::MOPRR, DL, XLenVT, Op.getOperand(1),
                        Op.getOperand(2), Op.getOperand(3));
   }
+  case Intrinsic::clmul:
   case Intrinsic::riscv_clmul:
     return DAG.getNode(RISCVISD::CLMUL, DL, XLenVT, Op.getOperand(1),
                        Op.getOperand(2));
@@ -14283,6 +14284,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
       return;
     }
+    case Intrinsic::clmul:
     case Intrinsic::riscv_clmul: {
       if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
         return;

>From 1717aac5530bb180fd180801c0848bfdde5c25a1 Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscardssmith at gmail.com>
Date: Sun, 18 May 2025 09:13:34 -0400
Subject: [PATCH 2/8] --amend

---
 llvm/docs/LangRef.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 1444654a248e2..fc1daa2f2adbd 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -18093,7 +18093,7 @@ on any integer bit width or vectors of integers.
 Overview
 """""""""
 
-The '``llvm.clmul``' family of intrinsics functions perform carryless multiplication
+The '``llvm.clmul``' family of intrinsic functions performs carryless multiplication
 (also known as xor multiplication) on the 2 arguments.
 
 Arguments

>From d9d04fe9a80e59dd4071146933faac0f073d59c3 Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscardssmith at gmail.com>
Date: Tue, 20 May 2025 00:54:25 -0400
Subject: [PATCH 3/8] teach selection dag about clmul legalization

---
 llvm/include/llvm/CodeGen/TargetLowering.h    |  5 +++
 llvm/lib/CodeGen/IntrinsicLowering.cpp        |  2 +-
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |  4 +++
 .../CodeGen/SelectionDAG/TargetLowering.cpp   | 31 +++++++++++++++++++
 llvm/lib/CodeGen/TargetLoweringBase.cpp       |  3 ++
 llvm/lib/Target/RISCV/RISCVInstrInfoZb.td     |  1 +
 6 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 03099e9ad44dc..fd773e7643dbe 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -5375,6 +5375,11 @@ class TargetLowering : public TargetLoweringBase {
   /// \returns The expansion if successful, SDValue() otherwise
   SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const;
 
+  /// Expand carryless multiply.
+  /// \param N Node to expand
+  /// \returns The expansion if successful, SDValue() otherwise
+  SDValue expandCLMUL(SDNode *N, SelectionDAG &DAG) const;
+
   /// Expand rotations.
   /// \param N Node to expand
   /// \param AllowVectorOps expand vector rotate, this should only be performed
diff --git a/llvm/lib/CodeGen/IntrinsicLowering.cpp b/llvm/lib/CodeGen/IntrinsicLowering.cpp
index d66544ee87ea4..8e6020d1055e9 100644
--- a/llvm/lib/CodeGen/IntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/IntrinsicLowering.cpp
@@ -215,7 +215,7 @@ static Value *LowerCLMUL(LLVMContext &Context, Value *V1, Value *V2, Instruction
     V1 = Builder.CreateLShr(V1, One, "clmul.V1");
     V2 = Builder.CreateShl(V2, One, "clmul.V2");
   }
-  return LowerCTPOP(Context, Res, IP);
+  return Res;
 }
 
 static void ReplaceFPIntrinsicWithCall(CallInst *CI, const char *Fname,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 528c07cc5549d..8dee5d1d769b0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3907,6 +3907,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
       Results.push_back(Expanded);
     break;
+  case ISD::CLMUL:
+    if (SDValue Expanded = TLI.expandCLMUL(Node, DAG))
+      Results.push_back(Expanded);
+    break;
   case ISD::ROTL:
   case ISD::ROTR:
     if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index da999b5057d49..071e560fc8a96 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8131,6 +8131,37 @@ SDValue TargetLowering::expandFunnelShift(SDNode *Node,
   return DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
 }
 
+SDValue TargetLowering::expandCLMUL(SDNode *Node,
+                                          SelectionDAG &DAG) const {
+  SDLoc DL(Node);
+  EVT VT = Node->getValueType(0);
+  SDValue V1 = Node->getOperand(0);
+  SDValue V2 = Node->getOperand(1);
+  unsigned NumBitsPerElt = VT.getScalarSizeInBits();
+
+  // Only expand vector types if we have the appropriate vector bit operations.
+  // This includes the operations needed to expand CTPOP if it isn't supported.
+  if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
+                        (!isOperationLegalOrCustom(ISD::SRL, VT) ||
+                        !isOperationLegalOrCustom(ISD::SHL, VT) ||
+                        !isOperationLegalOrCustom(ISD::XOR, VT) ||
+                        !isOperationLegalOrCustom(ISD::AND, VT) ||
+                        !isOperationLegalOrCustomOrPromote(ISD::OR, VT))))
+    return SDValue();
+
+  SDValue Res = DAG.getConstant(0, DL, VT);
+  SDValue Zero = DAG.getConstant(0, DL, VT);
+  SDValue One = DAG.getConstant(1, DL, VT);
+  for (unsigned i = 0; i < NumBitsPerElt; ++i) {
+    SDValue LowBit = DAG.getNode(ISD::AND, DL, VT, V1, One);
+    SDValue Pred = DAG.getNode(ISD::SELECT, DL, VT, LowBit, V2, Zero);
+    Res = DAG.getNode(ISD::XOR, DL, VT, Res, Pred);
+    V1 = DAG.getNode(ISD::SRL, DL, VT, V1, One);
+    V2 = DAG.getNode(ISD::SHL, DL, VT, V2, One);
+  }
+  return Res;
+}
+
 // TODO: Merge with expandFunnelShift.
 SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
                                   SelectionDAG &DAG) const {
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index c85f0c71ef25f..8df1c863d2d6e 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -781,6 +781,9 @@ void TargetLoweringBase::initActions() {
     // Absolute difference
     setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Expand);
 
+    // Carryless multiply
+    setOperationAction(ISD::CLMUL, VT, Expand);
+
     // Saturated trunc
     setOperationAction(ISD::TRUNCATE_SSAT_S, VT, Expand);
     setOperationAction(ISD::TRUNCATE_SSAT_U, VT, Expand);
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index 4353e94bdb1d0..fbd227922bd72 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -794,6 +794,7 @@ def : Sh3AddPat<SH3ADD>;
 } // Predicates = [HasStdExtZba, IsRV64]
 
 let Predicates = [HasStdExtZbcOrZbkc] in {
+def : PatGprGpr<clmul, CLMUL>;
 def : PatGprGpr<riscv_clmul, CLMUL>;
 def : PatGprGpr<riscv_clmulh, CLMULH>;
 } // Predicates = [HasStdExtZbcOrZbkc]

>From 566ff04b094502c207c3901e061796da479c4460 Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscardssmith at gmail.com>
Date: Tue, 20 May 2025 01:29:35 -0400
Subject: [PATCH 4/8] fix

---
 llvm/lib/CodeGen/IntrinsicLowering.cpp    | 2 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoZb.td | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/IntrinsicLowering.cpp b/llvm/lib/CodeGen/IntrinsicLowering.cpp
index 8e6020d1055e9..a8c85469086b8 100644
--- a/llvm/lib/CodeGen/IntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/IntrinsicLowering.cpp
@@ -204,7 +204,7 @@ static Value *LowerCLMUL(LLVMContext &Context, Value *V1, Value *V2, Instruction
 
   IRBuilder<> Builder(IP);
 
-  unsigned BitSize = V1->getType()->getPrimitiveSizeInBits();
+  unsigned BitSize = V1->getType()->getScalarSizeInBits();
   Value *Res = ConstantInt::get(V1->getType(), 0);
   Value *Zero = ConstantInt::get(V1->getType(), 0);
   Value *One = ConstantInt::get(V1->getType(), 1);
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index fbd227922bd72..97fdbe85417fe 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -54,6 +54,7 @@ def riscv_unzip   : RVSDNode<"UNZIP",   SDTIntUnaryOp>;
 def riscv_absw    : RVSDNode<"ABSW",    SDTIntUnaryOp>;
 
 // Scalar cryptography
+def clmul         : SDNode<"CLMUL",   SDTIntBinOp>;
 def riscv_clmul   : RVSDNode<"CLMUL",   SDTIntBinOp>;
 def riscv_clmulh  : RVSDNode<"CLMULH",  SDTIntBinOp>;
 def riscv_clmulr  : RVSDNode<"CLMULR",  SDTIntBinOp>;

>From 27c09ab048ee9dee4295f1f0961e6f235883e20b Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscardssmith at gmail.com>
Date: Tue, 20 May 2025 14:23:34 -0400
Subject: [PATCH 5/8] fixes

---
 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 16 ++++++++++++----
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp      |  6 ++++--
 llvm/lib/Target/RISCV/RISCVInstrInfoZb.td        |  2 +-
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 071e560fc8a96..899b66478fe10 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8132,33 +8132,41 @@ SDValue TargetLowering::expandFunnelShift(SDNode *Node,
 }
 
 SDValue TargetLowering::expandCLMUL(SDNode *Node,
-                                          SelectionDAG &DAG) const {
+                                    SelectionDAG &DAG) const {
   SDLoc DL(Node);
   EVT VT = Node->getValueType(0);
   SDValue V1 = Node->getOperand(0);
   SDValue V2 = Node->getOperand(1);
   unsigned NumBitsPerElt = VT.getScalarSizeInBits();
 
+  EVT SetCCType =
+      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
   // Only expand vector types if we have the appropriate vector bit operations.
-  // This includes the operations needed to expand CTPOP if it isn't supported.
   if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
                         (!isOperationLegalOrCustom(ISD::SRL, VT) ||
                         !isOperationLegalOrCustom(ISD::SHL, VT) ||
                         !isOperationLegalOrCustom(ISD::XOR, VT) ||
                         !isOperationLegalOrCustom(ISD::AND, VT) ||
+                        !isOperationLegalOrCustom(ISD::SELECT, VT) ||
                         !isOperationLegalOrCustomOrPromote(ISD::OR, VT))))
     return SDValue();
 
   SDValue Res = DAG.getConstant(0, DL, VT);
   SDValue Zero = DAG.getConstant(0, DL, VT);
   SDValue One = DAG.getConstant(1, DL, VT);
-  for (unsigned i = 0; i < NumBitsPerElt; ++i) {
+  for (unsigned i = 0; i < NumBitsPerElt-1; ++i) {
     SDValue LowBit = DAG.getNode(ISD::AND, DL, VT, V1, One);
-    SDValue Pred = DAG.getNode(ISD::SELECT, DL, VT, LowBit, V2, Zero);
+    SDValue LowBool = DAG.getSetCC(DL, SetCCType, LowBit, One, ISD::SETULT);
+    SDValue Pred = DAG.getNode(ISD::SELECT, DL, VT, LowBool, V2, Zero);
     Res = DAG.getNode(ISD::XOR, DL, VT, Res, Pred);
     V1 = DAG.getNode(ISD::SRL, DL, VT, V1, One);
     V2 = DAG.getNode(ISD::SHL, DL, VT, V2, One);
   }
+  // unroll last iteration to prevent dead nodes
+  SDValue LowBit = DAG.getNode(ISD::AND, DL, VT, V1, One);
+  SDValue LowBool = DAG.getSetCC(DL, SetCCType, LowBit, One, ISD::SETULT);
+  SDValue Pred = DAG.getNode(ISD::SELECT, DL, VT, LowBool, V2, Zero);
+  Res = DAG.getNode(ISD::XOR, DL, VT, Res, Pred);
   return Res;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 470685b9c68bb..5fe5f42b65fdd 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -397,6 +397,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                        Legal);
   }
 
+  if (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc()) {
+    setOperationAction(ISD::CLMUL, XLenVT, Legal);
+  }
+
   if (Subtarget.hasStdExtZbb() ||
       (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())) {
     if (Subtarget.is64Bit())
@@ -10347,7 +10351,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     return DAG.getNode(RISCVISD::MOPRR, DL, XLenVT, Op.getOperand(1),
                        Op.getOperand(2), Op.getOperand(3));
   }
-  case Intrinsic::clmul:
   case Intrinsic::riscv_clmul:
     return DAG.getNode(RISCVISD::CLMUL, DL, XLenVT, Op.getOperand(1),
                        Op.getOperand(2));
@@ -14284,7 +14287,6 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
       return;
     }
-    case Intrinsic::clmul:
     case Intrinsic::riscv_clmul: {
       if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
         return;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index 97fdbe85417fe..2291dc9fb79eb 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -54,7 +54,7 @@ def riscv_unzip   : RVSDNode<"UNZIP",   SDTIntUnaryOp>;
 def riscv_absw    : RVSDNode<"ABSW",    SDTIntUnaryOp>;
 
 // Scalar cryptography
-def clmul         : SDNode<"CLMUL",   SDTIntBinOp>;
+def clmul         : RVSDNode<"CLMUL",   SDTIntBinOp>;
 def riscv_clmul   : RVSDNode<"CLMUL",   SDTIntBinOp>;
 def riscv_clmulh  : RVSDNode<"CLMULH",  SDTIntBinOp>;
 def riscv_clmulr  : RVSDNode<"CLMULR",  SDTIntBinOp>;

>From 219908551162074a9e4b3ee7d7ef7bdd62241048 Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscardssmith at gmail.com>
Date: Tue, 20 May 2025 14:26:04 -0400
Subject: [PATCH 6/8] remvoe int_vp_clmul

---
 llvm/include/llvm/IR/Intrinsics.td | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 1857829910340..cc6130cf41e82 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2105,12 +2105,6 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
                                LLVMMatchType<0>,
                                LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                llvm_i32_ty]>;
-  def int_vp_clmul : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
-                             [ LLVMMatchType<0>,
-                               LLVMMatchType<0>,
-                               LLVMMatchType<0>,
-                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
-                               llvm_i32_ty]>;
   def int_vp_sadd_sat : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
                              [ LLVMMatchType<0>,
                                LLVMMatchType<0>,

>From 3b52afd1aa060031889744bbe45df1205c38f8d9 Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscardssmith at gmail.com>
Date: Tue, 20 May 2025 21:39:09 -0400
Subject: [PATCH 7/8] finish hooking up CLMUL to selectiondag?

---
 llvm/docs/LangRef.rst                         |  2 +-
 llvm/include/llvm/IR/Intrinsics.td            |  2 +-
 llvm/lib/CodeGen/IntrinsicLowering.cpp        |  2 +-
 .../SelectionDAG/LegalizeIntegerTypes.cpp     | 36 ++++++++++++++++++-
 llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h |  1 +
 .../SelectionDAG/LegalizeVectorTypes.cpp      |  3 ++
 .../CodeGen/SelectionDAG/TargetLowering.cpp   | 20 +++++------
 7 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index fc1daa2f2adbd..33ddb4f18e322 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -18107,7 +18107,7 @@ Semantics:
 """"""""""
 
 The ‘llvm.clmul’ intrinsic computes carryless multiply of ``%a`` and ``%b``, which is the result
-of applying the standard multiplication algorithm if you replace all of the aditions with exclusive ors.
+of applying the standard multiplication algorithm if you replace all of the additions with exclusive ors.
 The vector intrinsics, such as llvm.clmul.v4i32, operate on a per-element basis and the element order is not affected.
 
 Examples
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index cc6130cf41e82..5300101bcde01 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1432,7 +1432,7 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn] in {
   def int_fshr : DefaultAttrsIntrinsic<[llvm_anyint_ty],
       [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
   def int_clmul : DefaultAttrsIntrinsic<[llvm_anyint_ty],
-      [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
+      [LLVMMatchType<0>, LLVMMatchType<0>]>;
 }
 
 let IntrProperties = [IntrNoMem, IntrSpeculatable, IntrWillReturn,
diff --git a/llvm/lib/CodeGen/IntrinsicLowering.cpp b/llvm/lib/CodeGen/IntrinsicLowering.cpp
index a8c85469086b8..9111790e0193b 100644
--- a/llvm/lib/CodeGen/IntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/IntrinsicLowering.cpp
@@ -208,7 +208,7 @@ static Value *LowerCLMUL(LLVMContext &Context, Value *V1, Value *V2, Instruction
   Value *Res = ConstantInt::get(V1->getType(), 0);
   Value *Zero = ConstantInt::get(V1->getType(), 0);
   Value *One = ConstantInt::get(V1->getType(), 1);
-  for (unsigned I = 1; I < BitSize; I ++) {
+  for (unsigned I = 1; I < BitSize; I++) {
     Value *LowBit = Builder.CreateAnd(V1, One, "clmul.isodd");
     Value *Pred = Builder.CreateSelect(LowBit, V2, Zero, "clmul.V2_or_zero");
     Res = Builder.CreateXor(Res, Pred, "clmul.Res");
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 90af5f2cd8e70..11d8749cfa8bf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -207,7 +207,8 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::VP_XOR:
   case ISD::VP_ADD:
   case ISD::VP_SUB:
-  case ISD::VP_MUL:      Res = PromoteIntRes_SimpleIntBinOp(N); break;
+  case ISD::VP_MUL:
+  case ISD::CLMUL:      Res = PromoteIntRes_SimpleIntBinOp(N); break;
 
   case ISD::ABDS:
   case ISD::AVGCEILS:
@@ -3090,6 +3091,10 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
     ExpandIntRes_FunnelShift(N, Lo, Hi);
     break;
 
+  case ISD::CLMUL:
+    ExpandIntRes_CLMUL(N, Lo, Hi);
+    break;
+
   case ISD::VSCALE:
     ExpandIntRes_VSCALE(N, Lo, Hi);
     break;
@@ -5417,6 +5422,35 @@ void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo,
   Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt);
 }
 
+void DAGTypeLegalizer::ExpandIntRes_CLMUL(SDNode *N, SDValue &Lo,
+                                                SDValue &Hi) {
+  // Values numbered from least significant to most significant.
+  SDValue In1, In2, In3, In4;
+  GetExpandedInteger(N->getOperand(0), In3, In4);
+  GetExpandedInteger(N->getOperand(1), In1, In2);
+  EVT HalfVT = In1.getValueType();
+  SDLoc DL(N);
+  
+  // CLMUL is carryless so Lo is computed from the low half
+  Lo = DAG.getNode(ISD::CLMUL, DL, HalfVT, In1, In3);
+  // the high bits not included in CLMUL(A,B) can be computed by
+  // BITREVERSE(CLMUL(BITREVERSE(A), BITREVERSE(B))) >> 1
+  // Therefore we can compute the 2 hi/lo cross products
+  // and the the overflow of the low product
+  // and xor them together to compute HI
+  SDValue BitRevIn1 = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, In1);
+  SDValue BitRevIn3 = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, In3);
+  SDValue BitRevLoHi = DAG.getNode(ISD::CLMUL, DL, HalfVT, BitRevIn1, BitRevIn3);
+  SDValue LoHi = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, BitRevLoHi);
+  SDValue One = DAG.getConstant(0, DL, HalfVT);
+  Hi = DAG.getNode(ISD::SRL, DL, HalfVT, LoHi, One);
+  
+  SDValue HITMP = DAG.getNode(ISD::CLMUL, DL, HalfVT, In1, In4);
+  Hi = DAG.getNode(ISD::XOR, DL, HalfVT, Hi, HITMP);
+  HITMP = DAG.getNode(ISD::CLMUL, DL, HalfVT, In2, In3);
+  Hi = DAG.getNode(ISD::XOR, DL, HalfVT, Hi, HITMP);
+}
+
 void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
                                            SDValue &Hi) {
   EVT VT = N->getValueType(0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index cf3a9e23f4878..255a587cb3d18 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -508,6 +508,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
 
   void ExpandIntRes_Rotate            (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_FunnelShift       (SDNode *N, SDValue &Lo, SDValue &Hi);
+  void ExpandIntRes_CLMUL             (SDNode *N, SDValue &Lo, SDValue &Hi);
 
   void ExpandIntRes_VSCALE            (SDNode *N, SDValue &Lo, SDValue &Hi);
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 4e9a6942d6cd9..8a73286425745 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -166,6 +166,7 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::SMAX:
   case ISD::UMIN:
   case ISD::UMAX:
+  case ISD::CLMUL:
 
   case ISD::SADDSAT:
   case ISD::UADDSAT:
@@ -1330,6 +1331,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::SMAX: case ISD::VP_SMAX:
   case ISD::UMIN: case ISD::VP_UMIN:
   case ISD::UMAX: case ISD::VP_UMAX:
+  case ISD::CLMUL:
   case ISD::SADDSAT: case ISD::VP_SADDSAT:
   case ISD::UADDSAT: case ISD::VP_UADDSAT:
   case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
@@ -4667,6 +4669,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
   case ISD::SSHLSAT:
   case ISD::USHLSAT:
+  case ISD::CLMUL:
   case ISD::ROTL:
   case ISD::ROTR:
   case ISD::AVGFLOORS:
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 899b66478fe10..1015fa943ba90 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8147,26 +8147,22 @@ SDValue TargetLowering::expandCLMUL(SDNode *Node,
                         !isOperationLegalOrCustom(ISD::SHL, VT) ||
                         !isOperationLegalOrCustom(ISD::XOR, VT) ||
                         !isOperationLegalOrCustom(ISD::AND, VT) ||
-                        !isOperationLegalOrCustom(ISD::SELECT, VT) ||
-                        !isOperationLegalOrCustomOrPromote(ISD::OR, VT))))
+                        !isOperationLegalOrCustom(ISD::SELECT, VT))))
     return SDValue();
 
   SDValue Res = DAG.getConstant(0, DL, VT);
   SDValue Zero = DAG.getConstant(0, DL, VT);
   SDValue One = DAG.getConstant(1, DL, VT);
-  for (unsigned i = 0; i < NumBitsPerElt-1; ++i) {
+  for (unsigned I = 0; I < NumBitsPerElt-1; ++I) {
     SDValue LowBit = DAG.getNode(ISD::AND, DL, VT, V1, One);
-    SDValue LowBool = DAG.getSetCC(DL, SetCCType, LowBit, One, ISD::SETULT);
+    SDValue LowBool = DAG.getSetCC(DL, SetCCType, LowBit, Zero, ISD::SETNE);
     SDValue Pred = DAG.getNode(ISD::SELECT, DL, VT, LowBool, V2, Zero);
     Res = DAG.getNode(ISD::XOR, DL, VT, Res, Pred);
-    V1 = DAG.getNode(ISD::SRL, DL, VT, V1, One);
-    V2 = DAG.getNode(ISD::SHL, DL, VT, V2, One);
-  }
-  // unroll last iteration to prevent dead nodes
-  SDValue LowBit = DAG.getNode(ISD::AND, DL, VT, V1, One);
-  SDValue LowBool = DAG.getSetCC(DL, SetCCType, LowBit, One, ISD::SETULT);
-  SDValue Pred = DAG.getNode(ISD::SELECT, DL, VT, LowBool, V2, Zero);
-  Res = DAG.getNode(ISD::XOR, DL, VT, Res, Pred);
+    if (I != NumBitsPerElt) {
+      V1 = DAG.getNode(ISD::SRL, DL, VT, V1, One);
+      V2 = DAG.getNode(ISD::SHL, DL, VT, V2, One);
+    }
+  }
   return Res;
 }
 

>From a1f106da702e05db274c0810f31aff735f66ed41 Mon Sep 17 00:00:00 2001
From: Oscar Smith <oscar.smith at juliacomputing.com>
Date: Wed, 21 May 2025 21:31:17 +0000
Subject: [PATCH 8/8] cleanup

---
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |  3 +--
 .../SelectionDAG/LegalizeIntegerTypes.cpp     | 22 +++++++++----------
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  6 ++++-
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |  4 ++--
 4 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 8dee5d1d769b0..2758f6f6349d6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3908,8 +3908,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       Results.push_back(Expanded);
     break;
   case ISD::CLMUL:
-    if (SDValue Expanded = TLI.expandCLMUL(Node, DAG))
-      Results.push_back(Expanded);
+    Results.push_back(TLI.expandCLMUL(Node, DAG));
     break;
   case ISD::ROTL:
   case ISD::ROTR:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 11d8749cfa8bf..4982955630b1a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -5425,29 +5425,29 @@ void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo,
 void DAGTypeLegalizer::ExpandIntRes_CLMUL(SDNode *N, SDValue &Lo,
                                                 SDValue &Hi) {
   // Values numbered from least significant to most significant.
-  SDValue In1, In2, In3, In4;
-  GetExpandedInteger(N->getOperand(0), In3, In4);
-  GetExpandedInteger(N->getOperand(1), In1, In2);
-  EVT HalfVT = In1.getValueType();
+  SDValue LL, LH, RL, RH;
+  GetExpandedInteger(N->getOperand(0), LL, LH);
+  GetExpandedInteger(N->getOperand(1), RL, RH);
+  EVT HalfVT = LL.getValueType();
   SDLoc DL(N);
   
   // CLMUL is carryless so Lo is computed from the low half
-  Lo = DAG.getNode(ISD::CLMUL, DL, HalfVT, In1, In3);
+  Lo = DAG.getNode(ISD::CLMUL, DL, HalfVT, LL, RL);
   // the high bits not included in CLMUL(A,B) can be computed by
   // BITREVERSE(CLMUL(BITREVERSE(A), BITREVERSE(B))) >> 1
   // Therefore we can compute the 2 hi/lo cross products
   // and the the overflow of the low product
   // and xor them together to compute HI
-  SDValue BitRevIn1 = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, In1);
-  SDValue BitRevIn3 = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, In3);
-  SDValue BitRevLoHi = DAG.getNode(ISD::CLMUL, DL, HalfVT, BitRevIn1, BitRevIn3);
+  SDValue BitRevLL = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, LL);
+  SDValue BitRevRL = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, RL);
+  SDValue BitRevLoHi = DAG.getNode(ISD::CLMUL, DL, HalfVT, BitRevLL, BitRevRL);
   SDValue LoHi = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, BitRevLoHi);
-  SDValue One = DAG.getConstant(0, DL, HalfVT);
+  SDValue One = DAG.getShiftAmountConstant(1, HalfVT, DL);
   Hi = DAG.getNode(ISD::SRL, DL, HalfVT, LoHi, One);
   
-  SDValue HITMP = DAG.getNode(ISD::CLMUL, DL, HalfVT, In1, In4);
+  SDValue HITMP = DAG.getNode(ISD::CLMUL, DL, HalfVT, LL, RH);
   Hi = DAG.getNode(ISD::XOR, DL, HalfVT, Hi, HITMP);
-  HITMP = DAG.getNode(ISD::CLMUL, DL, HalfVT, In2, In3);
+  HITMP = DAG.getNode(ISD::CLMUL, DL, HalfVT, LH, RL);
   Hi = DAG.getNode(ISD::XOR, DL, HalfVT, Hi, HITMP);
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index a4e350702ebc8..374bcb51e86ed 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7191,7 +7191,11 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   case Intrinsic::clmul: {
     SDValue Op1 = getValue(I.getArgOperand(0));
     SDValue Op2 = getValue(I.getArgOperand(1));
-    setValue(&I, DAG.getNode(ISD::CLMUL, sdl, Op1.getValueType(), Op1, Op2));
+    EVT VT =  Op1.getValueType();
+    assert(VT.isInteger() && "This operator does not apply to FP types!");
+    assert(Op1.getValueType() == Op2.getValueType() &&
+           Op1.getValueType() == VT && "Binary operator types must match!");
+    setValue(&I, DAG.getNode(ISD::CLMUL, sdl, VT, Op1, Op2));
     return;
   }
   case Intrinsic::sadd_sat: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 1015fa943ba90..cfd7bd48822d6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8153,12 +8153,12 @@ SDValue TargetLowering::expandCLMUL(SDNode *Node,
   SDValue Res = DAG.getConstant(0, DL, VT);
   SDValue Zero = DAG.getConstant(0, DL, VT);
   SDValue One = DAG.getConstant(1, DL, VT);
-  for (unsigned I = 0; I < NumBitsPerElt-1; ++I) {
+  for (unsigned I = 0; I < NumBitsPerElt; ++I) {
     SDValue LowBit = DAG.getNode(ISD::AND, DL, VT, V1, One);
     SDValue LowBool = DAG.getSetCC(DL, SetCCType, LowBit, Zero, ISD::SETNE);
     SDValue Pred = DAG.getNode(ISD::SELECT, DL, VT, LowBool, V2, Zero);
     Res = DAG.getNode(ISD::XOR, DL, VT, Res, Pred);
-    if (I != NumBitsPerElt) {
+    if (I != NumBitsPerElt-1) {
       V1 = DAG.getNode(ISD::SRL, DL, VT, V1, One);
       V2 = DAG.getNode(ISD::SHL, DL, VT, V2, One);
     }



More information about the llvm-commits mailing list