[llvm] [SPIR-V] Implement builtins for OpIAddCarry/OpISubBorrow and improve/fix type inference (PR #115192)

Vyacheslav Levytskyy via llvm-commits llvm-commits at lists.llvm.org
Tue Nov 12 03:59:31 PST 2024


https://github.com/VyacheslavLevytskyy updated https://github.com/llvm/llvm-project/pull/115192

>From 02bf2fd850e118e2c25ceed57c07ce587acd17ab Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Wed, 6 Nov 2024 09:18:01 -0800
Subject: [PATCH 1/5] call lowering may not know register's type

---
 llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 98cf598a1f031a..d914f3cf256d48 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -545,13 +545,23 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
       Register ArgReg = Arg.Regs[0];
       ArgVRegs.push_back(ArgReg);
       SPIRVType *SpvType = GR->getSPIRVTypeForVReg(ArgReg);
-      if (!SpvType) {
+      // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) we should
+      // wait with setting the type for the virtual register until pre-legalizer
+      // step when we access @llvm.spv.assign.ptr.type.p...(...)'s info.
+      if (!SpvType && !isUntypedPointerTy(Arg.Ty)) {
         SpvType = GR->getOrCreateSPIRVType(Arg.Ty, MIRBuilder);
         GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF);
       }
       if (!MRI->getRegClassOrNull(ArgReg)) {
-        MRI->setRegClass(ArgReg, GR->getRegClass(SpvType));
-        MRI->setType(ArgReg, GR->getRegType(SpvType));
+        // Either we have SpvType created, or Arg.Ty is an untyped pointer and
+        // we know its virtual register's class and type.
+        MRI->setRegClass(ArgReg, SpvType ? GR->getRegClass(SpvType)
+                                         : &SPIRV::pIDRegClass);
+        MRI->setType(
+            ArgReg,
+            SpvType ? GR->getRegType(SpvType)
+                    : LLT::pointer(cast<PointerType>(Arg.Ty)->getAddressSpace(),
+                                   GR->getPointerSize()));
       }
     }
     auto instructionSet = canUseOpenCL ? SPIRV::InstructionSet::OpenCL_std

>From 86837dff62ef5634ba95c373579243b6943ddd63 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Wed, 6 Nov 2024 10:46:13 -0800
Subject: [PATCH 2/5] ensure that correct types are applied to virtual
 registers which were used as arguments in call lowering and so caused early
 definition of SPIR-V types

---
 llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp   | 27 +++++++---
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp |  2 +
 ...operand-types-vs-calllowering-unwrapped.ll | 50 +++++++++++++++++++
 ...phi-valid-operand-types-vs-calllowering.ll | 49 ++++++++++++++++++
 4 files changed, 121 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index d914f3cf256d48..8d1b82465d3df2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -545,16 +545,29 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
       Register ArgReg = Arg.Regs[0];
       ArgVRegs.push_back(ArgReg);
       SPIRVType *SpvType = GR->getSPIRVTypeForVReg(ArgReg);
-      // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) we should
-      // wait with setting the type for the virtual register until pre-legalizer
-      // step when we access @llvm.spv.assign.ptr.type.p...(...)'s info.
-      if (!SpvType && !isUntypedPointerTy(Arg.Ty)) {
-        SpvType = GR->getOrCreateSPIRVType(Arg.Ty, MIRBuilder);
-        GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF);
+      if (!SpvType) {
+        Type *ArgTy = nullptr;
+        if (auto *PtrArgTy = dyn_cast<PointerType>(Arg.Ty)) {
+          // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) and we
+          // don't have access to original value in LLVM IR or info about
+          // deduced pointee type, then we should wait with setting the type for
+          // the virtual register until pre-legalizer step when we access
+          // @llvm.spv.assign.ptr.type.p...(...)'s info.
+          if (Arg.OrigValue)
+            if (Type *ElemTy = GR->findDeducedElementType(Arg.OrigValue))
+              ArgTy = TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace());
+        } else {
+          ArgTy = Arg.Ty;
+        }
+        if (ArgTy) {
+          SpvType = GR->getOrCreateSPIRVType(ArgTy, MIRBuilder);
+          GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF);
+        }
       }
       if (!MRI->getRegClassOrNull(ArgReg)) {
         // Either we have SpvType created, or Arg.Ty is an untyped pointer and
-        // we know its virtual register's class and type.
+        // we know its virtual register's class and type even if we don't know
+        // pointee type.
         MRI->setRegClass(ArgReg, SpvType ? GR->getRegClass(SpvType)
                                          : &SPIRV::pIDRegClass);
         MRI->setType(
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 8b7e9c48de6c75..191533a52cccd9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -1219,6 +1219,8 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
   SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(AddressSpace)};
   auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
   I->setOperand(OperandToReplace, PtrCastI);
+  // We need to set up a pointee type for the newly created spv_ptrcast.
+  buildAssignPtr(B, ExpectedElementType, PtrCastI);
 }
 
 void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll
new file mode 100644
index 00000000000000..09c0a92d596ce3
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll
@@ -0,0 +1,50 @@
+; The goal of the test case is to ensure that correct types are applied to virtual registers
+; which were used as arguments in call lowering and so caused early definition of SPIR-V types.
+
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%t_id = type { %t_arr }
+%t_arr = type { [1 x i64] }
+%t_bf16 = type { i16 }
+
+define weak_odr dso_local spir_kernel void @foo(ptr addrspace(1) align 4 %_arg_ERR, ptr byval(%t_id) align 8 %_arg_ERR3) {
+entry:
+  %FloatArray.i = alloca [4 x float], align 4
+  %BF16Array.i = alloca [4 x %t_bf16], align 2
+  %0 = load i64, ptr %_arg_ERR3, align 8
+  %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_ERR, i64 %0
+  %FloatArray.ascast.i = addrspacecast ptr %FloatArray.i to ptr addrspace(4)
+  %BF16Array.ascast.i = addrspacecast ptr %BF16Array.i to ptr addrspace(4)
+  call spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4) %FloatArray.ascast.i, ptr addrspace(4) %BF16Array.ascast.i)
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %for.inc.i, %entry
+  %lsr.iv1 = phi ptr [ %scevgep2, %for.inc.i ], [ %FloatArray.i, %entry ]
+  %lsr.iv = phi ptr addrspace(4) [ %scevgep, %for.inc.i ], [ %BF16Array.ascast.i, %entry ]
+  %i.0.i = phi i32 [ 0, %entry ], [ %inc.i, %for.inc.i ]
+  %cmp.i = icmp ult i32 %i.0.i, 4
+  br i1 %cmp.i, label %for.body.i, label %exit
+
+for.body.i:                                       ; preds = %for.cond.i
+  %1 = load float, ptr %lsr.iv1, align 4
+  %call.i.i = call spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2) %lsr.iv)
+  %cmp5.i = fcmp une float %1, %call.i.i
+  br i1 %cmp5.i, label %if.then.i, label %for.inc.i
+
+if.then.i:                                        ; preds = %for.body.i
+  store i32 1, ptr addrspace(1) %add.ptr.i, align 4
+  br label %for.inc.i
+
+for.inc.i:                                        ; preds = %if.then.i, %for.body.i
+  %inc.i = add nuw nsw i32 %i.0.i, 1
+  %scevgep = getelementptr i8, ptr addrspace(4) %lsr.iv, i64 2
+  %scevgep2 = getelementptr i8, ptr %lsr.iv1, i64 4
+  br label %for.cond.i
+
+exit:                                             ; preds = %for.cond.i
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p1.i64(ptr noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg)
+declare dso_local spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4), ptr addrspace(4))
+declare dso_local spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2))
diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll
new file mode 100644
index 00000000000000..a31638e0b87043
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll
@@ -0,0 +1,49 @@
+; The goal of the test case is to ensure that correct types are applied to virtual registers
+; which were used as arguments in call lowering and so caused early definition of SPIR-V types.
+
+; RUN: %if spirv-tools %{ llc -O2 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%t_id = type { %t_arr }
+%t_arr = type { [1 x i64] }
+%t_bf16 = type { i16 }
+
+define weak_odr dso_local spir_kernel void @foo(ptr addrspace(1) align 4 %_arg_ERR, ptr byval(%t_id) align 8 %_arg_ERR3) {
+entry:
+  %FloatArray.i = alloca [4 x float], align 4
+  %BF16Array.i = alloca [4 x %t_bf16], align 2
+  %0 = load i64, ptr %_arg_ERR3, align 8
+  %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_ERR, i64 %0
+  %FloatArray.ascast.i = addrspacecast ptr %FloatArray.i to ptr addrspace(4)
+  %BF16Array.ascast.i = addrspacecast ptr %BF16Array.i to ptr addrspace(4)
+  call spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4) %FloatArray.ascast.i, ptr addrspace(4) %BF16Array.ascast.i)
+  br label %for.cond.i
+
+for.cond.i:                                       ; preds = %for.inc.i, %entry
+  %i.0.i = phi i32 [ 0, %entry ], [ %inc.i, %for.inc.i ]
+  %cmp.i = icmp ult i32 %i.0.i, 4
+  br i1 %cmp.i, label %for.body.i, label %exit
+
+for.body.i:                                       ; preds = %for.cond.i
+  %idxprom.i = zext nneg i32 %i.0.i to i64
+  %arrayidx.i = getelementptr inbounds [4 x float], ptr %FloatArray.i, i64 0, i64 %idxprom.i
+  %1 = load float, ptr %arrayidx.i, align 4
+  %arrayidx4.i = getelementptr inbounds [4 x %t_bf16], ptr addrspace(4) %BF16Array.ascast.i, i64 0, i64 %idxprom.i
+  %call.i.i = call spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2) %arrayidx4.i)
+  %cmp5.i = fcmp une float %1, %call.i.i
+  br i1 %cmp5.i, label %if.then.i, label %for.inc.i
+
+if.then.i:                                        ; preds = %for.body.i
+  store i32 1, ptr addrspace(1) %add.ptr.i, align 4
+  br label %for.inc.i
+
+for.inc.i:                                        ; preds = %if.then.i, %for.body.i
+  %inc.i = add nuw nsw i32 %i.0.i, 1
+  br label %for.cond.i
+
+exit: ; preds = %for.cond.i
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p1.i64(ptr noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg)
+declare dso_local spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4), ptr addrspace(4))
+declare dso_local spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2))

>From cc22d56e9d779ebea51a083dfa64663232964f46 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Thu, 7 Nov 2024 02:53:37 -0800
Subject: [PATCH 3/5] clang-format

---
 llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 8d1b82465d3df2..3c5397319aaf21 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -555,7 +555,8 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
           // @llvm.spv.assign.ptr.type.p...(...)'s info.
           if (Arg.OrigValue)
             if (Type *ElemTy = GR->findDeducedElementType(Arg.OrigValue))
-              ArgTy = TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace());
+              ArgTy =
+                  TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace());
         } else {
           ArgTy = Arg.Ty;
         }

>From 07530bde7e662aba461693da8b6577d7c68bbc81 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Fri, 8 Nov 2024 03:27:04 -0800
Subject: [PATCH 4/5] implement OpIAddCarry, OpISubBorrow; improve type
 inference; fix access to erased from parent after visit instructions; fix
 validation of pointer types for frexp and lgamma_r; improve parsing of
 builtin names

---
 llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp       |  69 +++++++++-
 llvm/lib/Target/SPIRV/SPIRVBuiltins.td        |   5 +
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp |  48 +++++--
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h   |  18 +++
 llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp   |   7 +-
 .../Target/SPIRV/SPIRVInstructionSelector.cpp |  19 +--
 llvm/lib/Target/SPIRV/SPIRVUtils.cpp          |  20 +++
 llvm/lib/Target/SPIRV/SPIRVUtils.h            |   5 +
 llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll  | 129 ++++++++++++++++++
 llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll | 127 +++++++++++++++++
 10 files changed, 410 insertions(+), 37 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 9aa1c31db7d208..06a37f1f559d44 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -190,8 +190,14 @@ std::string lookupBuiltinNameHelper(StringRef DemangledCall) {
   // Check if the extracted name contains type information between angle
   // brackets. If so, the builtin is an instantiated template - needs to have
   // the information after angle brackets and return type removed.
-  if (BuiltinName.find('<') && BuiltinName.back() == '>') {
-    BuiltinName = BuiltinName.substr(0, BuiltinName.find('<'));
+  std::size_t Pos1 = BuiltinName.rfind('<');
+  if (Pos1 != std::string::npos && BuiltinName.back() == '>') {
+    std::size_t Pos2 = BuiltinName.rfind(' ', Pos1);
+    if (Pos2 == std::string::npos)
+      Pos2 = 0;
+    else
+      ++Pos2;
+    BuiltinName = BuiltinName.substr(Pos2, Pos1 - Pos2);
     BuiltinName = BuiltinName.substr(BuiltinName.find_last_of(' ') + 1);
   }
 
@@ -461,9 +467,11 @@ static Register buildBuiltinVariableLoad(
     SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, LLT LLType,
     Register Reg = Register(0), bool isConst = true, bool hasLinkageTy = true) {
   Register NewRegister =
-      MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);
-  MIRBuilder.getMRI()->setType(NewRegister,
-                               LLT::pointer(0, GR->getPointerSize()));
+      MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::pIDRegClass);
+  MIRBuilder.getMRI()->setType(
+      NewRegister,
+      LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function),
+                   GR->getPointerSize()));
   SPIRVType *PtrType = GR->getOrCreateSPIRVPointerType(
       VariableType, MIRBuilder, SPIRV::StorageClass::Input);
   GR->assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
@@ -1556,6 +1564,55 @@ static bool generateWaveInst(const SPIRV::IncomingCall *Call,
       /* isConst= */ false, /* hasLinkageTy= */ false);
 }
 
+// We expect a builtin
+//     Name(ptr sret([RetType]) %result, Type %operand1, Type %operand1)
+// where %result is a pointer to where the result of the builtin execution
+// is to be stored, and generate the following instructions:
+//     Res = Opcode RetType Operand1 Operand1
+//     OpStore RetVariable Res
+static bool generateICarryBorrowInst(const SPIRV::IncomingCall *Call,
+                                     MachineIRBuilder &MIRBuilder,
+                                     SPIRVGlobalRegistry *GR) {
+  const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
+  unsigned Opcode =
+      SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
+
+  Register SRetReg = Call->Arguments[0];
+  SPIRVType *PtrRetType = GR->getSPIRVTypeForVReg(SRetReg);
+  SPIRVType *RetType = GR->getPointeeType(PtrRetType);
+  if (!RetType)
+    report_fatal_error("The first parameter must be a pointer");
+  if (RetType->getOpcode() != SPIRV::OpTypeStruct)
+    report_fatal_error("Expected struct type result for the arithmetic with "
+                       "overflow builtins");
+
+  SPIRVType *OpType1 = GR->getSPIRVTypeForVReg(Call->Arguments[1]);
+  SPIRVType *OpType2 = GR->getSPIRVTypeForVReg(Call->Arguments[2]);
+  if (!OpType1 || !OpType2 || OpType1 != OpType2)
+    report_fatal_error("Operands must have the same type");
+  if (OpType1->getOpcode() == SPIRV::OpTypeVector)
+    switch (Opcode) {
+    case SPIRV::OpIAddCarryS:
+      Opcode = SPIRV::OpIAddCarryV;
+      break;
+    case SPIRV::OpISubBorrowS:
+      Opcode = SPIRV::OpISubBorrowV;
+      break;
+    }
+
+  MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+  Register ResReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
+  MRI->setRegClass(ResReg, &SPIRV::iIDRegClass);
+  GR->assignSPIRVTypeToVReg(RetType, ResReg, MIRBuilder.getMF());
+  MIRBuilder.buildInstr(Opcode)
+      .addDef(ResReg)
+      .addUse(GR->getSPIRVTypeID(RetType))
+      .addUse(Call->Arguments[1])
+      .addUse(Call->Arguments[2]);
+  MIRBuilder.buildInstr(SPIRV::OpStore).addUse(SRetReg).addUse(ResReg);
+  return true;
+}
+
 static bool generateGetQueryInst(const SPIRV::IncomingCall *Call,
                                  MachineIRBuilder &MIRBuilder,
                                  SPIRVGlobalRegistry *GR) {
@@ -2511,6 +2568,8 @@ std::optional<bool> lowerBuiltin(const StringRef DemangledCall,
     return generateDotOrFMulInst(Call.get(), MIRBuilder, GR);
   case SPIRV::Wave:
     return generateWaveInst(Call.get(), MIRBuilder, GR);
+  case SPIRV::ICarryBorrow:
+    return generateICarryBorrowInst(Call.get(), MIRBuilder, GR);
   case SPIRV::GetQuery:
     return generateGetQueryInst(Call.get(), MIRBuilder, GR);
   case SPIRV::ImageSizeQuery:
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index 29f11c3dd36865..1b95b1479bb932 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -63,6 +63,7 @@ def KernelClock : BuiltinGroup;
 def CastToPtr : BuiltinGroup;
 def Construct : BuiltinGroup;
 def CoopMatr : BuiltinGroup;
+def ICarryBorrow : BuiltinGroup;
 
 //===----------------------------------------------------------------------===//
 // Class defining a demangled builtin record. The information in the record
@@ -628,6 +629,10 @@ defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBar
 defm : DemangledNativeBuiltin<"work_group_barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
 defm : DemangledNativeBuiltin<"__spirv_ControlBarrier", OpenCL_std, Barrier, 3, 3, OpControlBarrier>;
 
+// ICarryBorrow builtin record:
+defm : DemangledNativeBuiltin<"__spirv_IAddCarry", OpenCL_std, ICarryBorrow, 3, 3, OpIAddCarryS>;
+defm : DemangledNativeBuiltin<"__spirv_ISubBorrow", OpenCL_std, ICarryBorrow, 3, 3, OpISubBorrowS>;
+
 // cl_intel_split_work_group_barrier
 defm : DemangledNativeBuiltin<"intel_work_group_barrier_arrive", OpenCL_std, Barrier, 1, 2, OpControlBarrierArriveINTEL>;
 defm : DemangledNativeBuiltin<"__spirv_ControlBarrierArriveINTEL", OpenCL_std, Barrier, 3, 3, OpControlBarrierArriveINTEL>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 191533a52cccd9..e6ef40e010dc20 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -76,8 +76,8 @@ class SPIRVEmitIntrinsics
   SPIRV::InstructionSet::InstructionSet InstrSet;
 
   // a register of Instructions that don't have a complete type definition
-  SmallPtrSet<Value *, 8> UncompleteTypeInfo;
-  SmallVector<Instruction *> PostprocessWorklist;
+  DenseMap<Value *, unsigned> UncompleteTypeInfo;
+  SmallVector<Value *> PostprocessWorklist;
 
   // well known result types of builtins
   enum WellKnownTypes { Event };
@@ -147,6 +147,7 @@ class SPIRVEmitIntrinsics
                                   std::unordered_set<Function *> &FVisited);
   void replaceWithPtrcasted(Instruction *CI, Type *NewElemTy, Type *KnownElemTy,
                             CallInst *AssignCI);
+  void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
 
   bool runOnFunction(Function &F);
   bool postprocessTypes();
@@ -272,6 +273,27 @@ static inline void reportFatalOnTokenType(const Instruction *I) {
                        false);
 }
 
+void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
+                                             bool DeleteOld) {
+  Src->replaceAllUsesWith(Dest);
+  // Update deduced type records
+  GR->updateIfExistDeducedElementType(Src, Dest, DeleteOld);
+  GR->updateIfExistAssignPtrTypeInstr(Src, Dest, DeleteOld);
+  // Update uncomplete type records if any
+  auto It = UncompleteTypeInfo.find(Src);
+  if (It == UncompleteTypeInfo.end())
+    return;
+  if (DeleteOld) {
+    unsigned Pos = It->second;
+    UncompleteTypeInfo.erase(Src);
+    UncompleteTypeInfo[Dest] = Pos;
+    PostprocessWorklist[Pos] = Dest;
+  } else {
+    UncompleteTypeInfo[Dest] = PostprocessWorklist.size();
+    PostprocessWorklist.push_back(Dest);
+  }
+}
+
 static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
   return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
          isPointerTy(SI->getValueOperand()->getType()) &&
@@ -434,7 +456,7 @@ void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
     if (!UnknownElemTypeI8)
       return;
     if (auto *I = dyn_cast<Instruction>(Op)) {
-      UncompleteTypeInfo.insert(I);
+      UncompleteTypeInfo[I] = PostprocessWorklist.size();
       PostprocessWorklist.push_back(I);
     }
   }
@@ -640,7 +662,7 @@ Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
   if (!UnknownElemTypeI8)
     return nullptr;
   if (auto *Instr = dyn_cast<Instruction>(I)) {
-    UncompleteTypeInfo.insert(Instr);
+    UncompleteTypeInfo[Instr] = PostprocessWorklist.size();
     PostprocessWorklist.push_back(Instr);
   }
   return IntegerType::getInt8Ty(I->getContext());
@@ -1062,7 +1084,7 @@ Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
                                      {I.getOperand(0)->getType()}, {Args});
   // remove switch to avoid its unneeded and undesirable unwrap into branches
   // and conditions
-  I.replaceAllUsesWith(NewI);
+  replaceAllUsesWith(&I, NewI);
   I.eraseFromParent();
   // insert artificial and temporary instruction to preserve valid CFG,
   // it will be removed after IR translation pass
@@ -1084,7 +1106,7 @@ Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
   for (auto &Op : I.operands())
     Args.push_back(Op);
   auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});
-  I.replaceAllUsesWith(NewI);
+  replaceAllUsesWith(&I, NewI);
   I.eraseFromParent();
   return NewI;
 }
@@ -1099,7 +1121,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
   // such bitcasts do not provide sufficient information, should be just skipped
   // here, and handled in insertPtrCastOrAssignTypeInstr.
   if (isPointerTy(I.getType())) {
-    I.replaceAllUsesWith(Source);
+    replaceAllUsesWith(&I, Source);
     I.eraseFromParent();
     return nullptr;
   }
@@ -1108,7 +1130,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
   SmallVector<Value *> Args(I.op_begin(), I.op_end());
   auto *NewI = B.CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args});
   std::string InstName = I.hasName() ? I.getName().str() : "";
-  I.replaceAllUsesWith(NewI);
+  replaceAllUsesWith(&I, NewI);
   I.eraseFromParent();
   NewI->setName(InstName);
   return NewI;
@@ -1333,7 +1355,7 @@ Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
   SmallVector<Value *> Args(I.op_begin(), I.op_end());
   auto *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});
   std::string InstName = I.hasName() ? I.getName().str() : "";
-  I.replaceAllUsesWith(NewI);
+  replaceAllUsesWith(&I, NewI);
   I.eraseFromParent();
   NewI->setName(InstName);
   return NewI;
@@ -1348,7 +1370,7 @@ SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
   SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
   auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});
   std::string InstName = I.hasName() ? I.getName().str() : "";
-  I.replaceAllUsesWith(NewI);
+  replaceAllUsesWith(&I, NewI);
   I.eraseFromParent();
   NewI->setName(InstName);
   return NewI;
@@ -1384,7 +1406,7 @@ Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
     Args.push_back(B.getInt32(Op));
   auto *NewI =
       B.CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args});
-  I.replaceAllUsesWith(NewI);
+  replaceAllUsesWith(&I, NewI);
   I.eraseFromParent();
   return NewI;
 }
@@ -1445,7 +1467,7 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
                                     {PtrTy, ArraySize->getType()}, {ArraySize})
                 : B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
   std::string InstName = I.hasName() ? I.getName().str() : "";
-  I.replaceAllUsesWith(NewI);
+  replaceAllUsesWith(&I, NewI);
   I.eraseFromParent();
   NewI->setName(InstName);
   return NewI;
@@ -1615,7 +1637,7 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
     auto *NewOp =
         buildIntrWithMD(Intrinsic::spv_track_constant,
                         {II->getType(), II->getType()}, t->second, I, {}, B);
-    I->replaceAllUsesWith(NewOp);
+    replaceAllUsesWith(I, NewOp, false);
     NewOp->setArgOperand(0, I);
   }
   bool IsPhi = isa<PHINode>(I), BPrepared = false;
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index a95b488960c4c3..2805300ae4437d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -180,6 +180,15 @@ class SPIRVGlobalRegistry {
     auto It = AssignPtrTypeInstr.find(Val);
     return It == AssignPtrTypeInstr.end() ? nullptr : It->second;
   }
+  // - Find a record and update its key or add a new record, if found.
+  void updateIfExistAssignPtrTypeInstr(Value *OldVal, Value *NewVal,
+                                       bool DeleteOld) {
+    if (CallInst *CI = findAssignPtrTypeInstr(OldVal)) {
+      if (DeleteOld)
+        AssignPtrTypeInstr.erase(OldVal);
+      AssignPtrTypeInstr[NewVal] = CI;
+    }
+  }
 
   // A registry of mutated values
   // (see `SPIRVPrepareFunctions::removeAggregateTypesFromSignature()`):
@@ -214,6 +223,15 @@ class SPIRVGlobalRegistry {
     auto It = DeducedElTys.find(Val);
     return It == DeducedElTys.end() ? nullptr : It->second;
   }
+  // - Find a record and update its key or add a new record, if found.
+  void updateIfExistDeducedElementType(Value *OldVal, Value *NewVal,
+                                       bool DeleteOld) {
+    if (Type *Ty = findDeducedElementType(OldVal)) {
+      if (DeleteOld)
+        DeducedElTys.erase(OldVal);
+      DeducedElTys[NewVal] = Ty;
+    }
+  }
   // - Add a record to the map of deduced composite types.
   void addDeducedCompositeType(Value *Val, Type *Ty) {
     DeducedNestedTys[Val] = Ty;
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index ecbceb5b472fa1..d9a79d7e525b4b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -473,8 +473,11 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
             MI.getOperand(2).getImm() != SPIRV::InstructionSet::OpenCL_std)
           continue;
         switch (MI.getOperand(3).getImm()) {
+        case SPIRV::OpenCLExtInst::frexp:
+        case SPIRV::OpenCLExtInst::lgamma_r:
         case SPIRV::OpenCLExtInst::remquo: {
-          // The last operand must be of a pointer to the return type.
+          // The last operand must be of a pointer to i32 or vector of i32
+          // values.
           MachineIRBuilder MIB(MI);
           SPIRVType *Int32Type = GR.getOrCreateSPIRVIntegerType(32, MIB);
           SPIRVType *RetType = MRI->getVRegDef(MI.getOperand(1).getReg());
@@ -487,8 +490,6 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
                         Int32Type, RetType->getOperand(2).getImm(), MIB));
         } break;
         case SPIRV::OpenCLExtInst::fract:
-        case SPIRV::OpenCLExtInst::frexp:
-        case SPIRV::OpenCLExtInst::lgamma_r:
         case SPIRV::OpenCLExtInst::modf:
         case SPIRV::OpenCLExtInst::sincos:
           // The last operand must be of a pointer to the base type represented
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index a968b65c173067..f9bfb80d6eede2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -3051,22 +3051,9 @@ bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
                                                 MachineInstr &I) const {
   // Change order of instructions if needed: all OpVariable instructions in a
   // function must be the first instructions in the first block
-  MachineFunction *MF = I.getParent()->getParent();
-  MachineBasicBlock *MBB = &MF->front();
-  auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end();
-  bool IsHeader = false;
-  unsigned Opcode;
-  for (; It != E && It != I; ++It) {
-    Opcode = It->getOpcode();
-    if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {
-      IsHeader = true;
-    } else if (IsHeader &&
-               !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {
-      ++It;
-      break;
-    }
-  }
-  return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable))
+  auto It = getOpVariableMBBIt(I);
+  return BuildMI(*It->getParent(), It, It->getDebugLoc(),
+                 TII.get(SPIRV::OpVariable))
       .addDef(ResVReg)
       .addUse(GR.getSPIRVTypeID(ResType))
       .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index f9b361e163c909..aeb2c29f7b8618 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -162,6 +162,26 @@ void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
   }
 }
 
+MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I) {
+  MachineFunction *MF = I.getParent()->getParent();
+  MachineBasicBlock *MBB = &MF->front();
+  MachineBasicBlock::iterator It = MBB->SkipPHIsAndLabels(MBB->begin()),
+                              E = MBB->end();
+  bool IsHeader = false;
+  unsigned Opcode;
+  for (; It != E && It != I; ++It) {
+    Opcode = It->getOpcode();
+    if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {
+      IsHeader = true;
+    } else if (IsHeader &&
+               !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {
+      ++It;
+      break;
+    }
+  }
+  return It;
+}
+
 SPIRV::StorageClass::StorageClass
 addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
   switch (AddrSpace) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index 11fd3a5c61dcae..298b0b93b0e4d2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -15,6 +15,7 @@
 
 #include "MCTargetDesc/SPIRVBaseInfo.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/TypedPointerType.h"
@@ -139,6 +140,10 @@ void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII,
 void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder,
                              const MDNode *GVarMD);
 
+// Return a valid position for the OpVariable instruction inside a function,
+// i.e., at the beginning of the first block of the function.
+MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I);
+
 // Convert a SPIR-V storage class to the corresponding LLVM IR address space.
 // TODO: maybe the following two functions should be handled in the subtarget
 // to allow for different OpenCL vs Vulkan handling.
diff --git a/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll b/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll
new file mode 100644
index 00000000000000..8f14eba21b63a2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll
@@ -0,0 +1,129 @@
+; Adapted from Khronos Translator test suite: test/iaddcarry_builtin.ll
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%i8struct = type {i8, i8}
+%i16struct = type {i16, i16}
+%i32struct = type {i32, i32}
+%i64struct = type {i64, i64}
+%vecstruct = type {<4 x i32>, <4 x i32>}
+
+; CHECK-SPIRV-DAG:                     [[uchar:%[a-z0-9_]+]] = OpTypeInt 8
+; CHECK-SPIRV-DAG:                    [[ushort:%[a-z0-9_]+]] = OpTypeInt 16
+; CHECK-SPIRV-DAG:                      [[uint:%[a-z0-9_]+]] = OpTypeInt 32
+; CHECK-SPIRV-DAG:                     [[ulong:%[a-z0-9_]+]] = OpTypeInt 64
+; CHECK-SPIRV-DAG:                      [[void:%[a-z0-9_]+]] = OpTypeVoid
+; CHECK-SPIRV-DAG:                  [[i8struct:%[a-z0-9_]+]] = OpTypeStruct [[uchar]] [[uchar]]
+; CHECK-SPIRV-DAG:    [[_ptr_Function_i8struct:%[a-z0-9_]+]] = OpTypePointer Function [[i8struct]]
+; CHECK-SPIRV-DAG:                 [[i16struct:%[a-z0-9_]+]] = OpTypeStruct [[ushort]] [[ushort]]
+; CHECK-SPIRV-DAG:   [[_ptr_Function_i16struct:%[a-z0-9_]+]] = OpTypePointer Function [[i16struct]]
+; CHECK-SPIRV-DAG:                 [[i32struct:%[a-z0-9_]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG:   [[_ptr_Function_i32struct:%[a-z0-9_]+]] = OpTypePointer Function [[i32struct]]
+; CHECK-SPIRV-DAG:                 [[i64struct:%[a-z0-9_]+]] = OpTypeStruct [[ulong]] [[ulong]]
+; CHECK-SPIRV-DAG:   [[_ptr_Function_i64struct:%[a-z0-9_]+]] = OpTypePointer Function [[i64struct]]
+; CHECK-SPIRV-DAG:                    [[v4uint:%[a-z0-9_]+]] = OpTypeVector [[uint]] 4
+; CHECK-SPIRV-DAG:                 [[vecstruct:%[a-z0-9_]+]] = OpTypeStruct [[v4uint]] [[v4uint]]
+; CHECK-SPIRV-DAG:   [[_ptr_Function_vecstruct:%[a-z0-9_]+]] = OpTypePointer Function [[vecstruct]]
+; CHECK-SPIRV-DAG:               [[struct_anon:%[a-z0-9_.]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_struct_anon:%[a-z0-9_]+]] = OpTypePointer Function [[struct_anon]]
+; CHECK-SPIRV-DAG:  [[_ptr_Generic_struct_anon:%[a-z0-9_]+]] = OpTypePointer Generic [[struct_anon]]
+
+define spir_func void @test_builtin_iaddcarrycc(i8 %a, i8 %b) {
+  entry:
+  %0 = alloca %i8struct
+  call void @_Z17__spirv_IAddCarrycc(ptr sret (%i8struct) %0, i8 %a, i8 %b)
+  ret void
+}
+; CHECK-SPIRV:           [[a:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV:           [[b:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV:       [[entry:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_11:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i8struct]] Function
+; CHECK-SPIRV:      [[var_12:%[a-z0-9_]+]] = OpIAddCarry [[i8struct]] [[a]] [[b]]
+; CHECK-SPIRV:                               OpStore [[var_11]] [[var_12]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryss(i16 %a, i16 %b) {
+  entry:
+  %0 = alloca %i16struct
+  call void @_Z17__spirv_IAddCarryss(ptr sret (%i16struct) %0, i16 %a, i16 %b)
+  ret void
+}
+; CHECK-SPIRV:         [[a_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV:         [[b_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV:     [[entry_0:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_21:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i16struct]] Function
+; CHECK-SPIRV:      [[var_22:%[a-z0-9_]+]] = OpIAddCarry [[i16struct]] [[a_0]] [[b_0]]
+; CHECK-SPIRV:                               OpStore [[var_21]] [[var_22]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryii(i32 %a, i32 %b) {
+  entry:
+  %0 = alloca %i32struct
+  call void @_Z17__spirv_IAddCarryii(ptr sret (%i32struct) %0, i32 %a, i32 %b)
+  ret void
+}
+; CHECK-SPIRV:         [[a_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV:         [[b_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV:     [[entry_1:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_31:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i32struct]] Function
+; CHECK-SPIRV:      [[var_32:%[a-z0-9_]+]] = OpIAddCarry [[i32struct]] [[a_1]] [[b_1]]
+; CHECK-SPIRV:                               OpStore [[var_31]] [[var_32]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryll(i64 %a, i64 %b) {
+  entry:
+  %0 = alloca %i64struct
+  call void @_Z17__spirv_IAddCarryll(ptr sret (%i64struct) %0, i64 %a, i64 %b)
+  ret void
+}
+; CHECK-SPIRV:         [[a_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV:         [[b_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV:     [[entry_2:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_41:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i64struct]] Function
+; CHECK-SPIRV:      [[var_42:%[a-z0-9_]+]] = OpIAddCarry [[i64struct]] [[a_2]] [[b_2]]
+; CHECK-SPIRV:                               OpStore [[var_41]] [[var_42]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+define spir_func void @test_builtin_iaddcarryDv4_xS_(<4 x i32> %a, <4 x i32> %b) {
+  entry:
+  %0 = alloca %vecstruct
+  call void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct) %0, <4 x i32> %a, <4 x i32> %b)
+  ret void
+}
+; CHECK-SPIRV:         [[a_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV:         [[b_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV:     [[entry_3:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_51:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_vecstruct]] Function
+; CHECK-SPIRV:      [[var_52:%[a-z0-9_]+]] = OpIAddCarry [[vecstruct]] [[a_3]] [[b_3]]
+; CHECK-SPIRV:                               OpStore [[var_51]] [[var_52]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+%struct.anon = type { i32, i32 }
+
+define spir_func void @test_builtin_iaddcarry_anon(i32 %a, i32 %b) {
+  entry:
+  %0 = alloca %struct.anon
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  call spir_func void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4 %1, i32 %a, i32 %b)
+  ret void
+}
+; CHECK-SPIRV:        [[a_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV:        [[b_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV:    [[entry_4:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:     [[var_59:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_struct_anon]] Function
+; CHECK-SPIRV:     [[var_61:%[a-z0-9_]+]] = OpPtrCastToGeneric [[_ptr_Generic_struct_anon]] [[var_59]]
+; CHECK-SPIRV:     [[var_62:%[a-z0-9_]+]] = OpIAddCarry [[struct_anon]] [[a_4]] [[b_4]]
+; CHECK-SPIRV:                              OpStore [[var_61]] [[var_62]]
+
+declare void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32)
+declare void @_Z17__spirv_IAddCarrycc(ptr sret(%i8struct), i8, i8)
+declare void @_Z17__spirv_IAddCarryss(ptr sret(%i16struct), i16, i16)
+declare void @_Z17__spirv_IAddCarryii(ptr sret(%i32struct), i32, i32)
+declare void @_Z17__spirv_IAddCarryll(ptr sret(%i64struct), i64, i64)
+declare void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct), <4 x i32>, <4 x i32>)
diff --git a/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll b/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll
new file mode 100644
index 00000000000000..08b4d2a1fa8e55
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll
@@ -0,0 +1,127 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+%i8struct = type {i8, i8}
+%i16struct = type {i16, i16}
+%i32struct = type {i32, i32}
+%i64struct = type {i64, i64}
+%vecstruct = type {<4 x i32>, <4 x i32>}
+
+; CHECK-SPIRV-DAG:                     [[uchar:%[a-z0-9_]+]] = OpTypeInt 8
+; CHECK-SPIRV-DAG:                    [[ushort:%[a-z0-9_]+]] = OpTypeInt 16
+; CHECK-SPIRV-DAG:                      [[uint:%[a-z0-9_]+]] = OpTypeInt 32
+; CHECK-SPIRV-DAG:                     [[ulong:%[a-z0-9_]+]] = OpTypeInt 64
+; CHECK-SPIRV-DAG:                      [[void:%[a-z0-9_]+]] = OpTypeVoid
+; CHECK-SPIRV-DAG:                  [[i8struct:%[a-z0-9_]+]] = OpTypeStruct [[uchar]] [[uchar]]
+; CHECK-SPIRV-DAG:    [[_ptr_Function_i8struct:%[a-z0-9_]+]] = OpTypePointer Function [[i8struct]]
+; CHECK-SPIRV-DAG:                 [[i16struct:%[a-z0-9_]+]] = OpTypeStruct [[ushort]] [[ushort]]
+; CHECK-SPIRV-DAG:   [[_ptr_Function_i16struct:%[a-z0-9_]+]] = OpTypePointer Function [[i16struct]]
+; CHECK-SPIRV-DAG:                 [[i32struct:%[a-z0-9_]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG:   [[_ptr_Function_i32struct:%[a-z0-9_]+]] = OpTypePointer Function [[i32struct]]
+; CHECK-SPIRV-DAG:                 [[i64struct:%[a-z0-9_]+]] = OpTypeStruct [[ulong]] [[ulong]]
+; CHECK-SPIRV-DAG:   [[_ptr_Function_i64struct:%[a-z0-9_]+]] = OpTypePointer Function [[i64struct]]
+; CHECK-SPIRV-DAG:                    [[v4uint:%[a-z0-9_]+]] = OpTypeVector [[uint]] 4
+; CHECK-SPIRV-DAG:                 [[vecstruct:%[a-z0-9_]+]] = OpTypeStruct [[v4uint]] [[v4uint]]
+; CHECK-SPIRV-DAG:   [[_ptr_Function_vecstruct:%[a-z0-9_]+]] = OpTypePointer Function [[vecstruct]]
+; CHECK-SPIRV-DAG:               [[struct_anon:%[a-z0-9_.]+]] = OpTypeStruct [[uint]] [[uint]]
+; CHECK-SPIRV-DAG: [[_ptr_Function_struct_anon:%[a-z0-9_]+]] = OpTypePointer Function [[struct_anon]]
+; CHECK-SPIRV-DAG:  [[_ptr_Generic_struct_anon:%[a-z0-9_]+]] = OpTypePointer Generic [[struct_anon]]
+
+define spir_func void @test_builtin_isubborrowcc(i8 %a, i8 %b) {
+  entry:
+  %0 = alloca %i8struct
+  call void @_Z18__spirv_ISubBorrowcc(ptr sret (%i8struct) %0, i8 %a, i8 %b)
+  ret void
+}
+; CHECK-SPIRV:           [[a:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV:           [[b:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]]
+; CHECK-SPIRV:       [[entry:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_11:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i8struct]] Function
+; CHECK-SPIRV:      [[var_12:%[a-z0-9_]+]] = OpISubBorrow [[i8struct]] [[a]] [[b]]
+; CHECK-SPIRV:                               OpStore [[var_11]] [[var_12]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowss(i16 %a, i16 %b) {
+  entry:
+  %0 = alloca %i16struct
+  call void @_Z18__spirv_ISubBorrowss(ptr sret (%i16struct) %0, i16 %a, i16 %b)
+  ret void
+}
+; CHECK-SPIRV:         [[a_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV:         [[b_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]]
+; CHECK-SPIRV:     [[entry_0:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_21:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i16struct]] Function
+; CHECK-SPIRV:      [[var_22:%[a-z0-9_]+]] = OpISubBorrow [[i16struct]] [[a_0]] [[b_0]]
+; CHECK-SPIRV:                               OpStore [[var_21]] [[var_22]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowii(i32 %a, i32 %b) {
+  entry:
+  %0 = alloca %i32struct
+  call void @_Z18__spirv_ISubBorrowii(ptr sret (%i32struct) %0, i32 %a, i32 %b)
+  ret void
+}
+; CHECK-SPIRV:         [[a_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV:         [[b_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV:     [[entry_1:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_31:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i32struct]] Function
+; CHECK-SPIRV:      [[var_32:%[a-z0-9_]+]] = OpISubBorrow [[i32struct]] [[a_1]] [[b_1]]
+; CHECK-SPIRV:                               OpStore [[var_31]] [[var_32]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowll(i64 %a, i64 %b) {
+  entry:
+  %0 = alloca %i64struct
+  call void @_Z18__spirv_ISubBorrowll(ptr sret (%i64struct) %0, i64 %a, i64 %b)
+  ret void
+}
+; CHECK-SPIRV:         [[a_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV:         [[b_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]]
+; CHECK-SPIRV:     [[entry_2:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_41:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i64struct]] Function
+; CHECK-SPIRV:      [[var_42:%[a-z0-9_]+]] = OpISubBorrow [[i64struct]] [[a_2]] [[b_2]]
+; CHECK-SPIRV:                               OpStore [[var_41]] [[var_42]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+define spir_func void @test_builtin_isubborrowDv4_xS_(<4 x i32> %a, <4 x i32> %b) {
+  entry:
+  %0 = alloca %vecstruct
+  call void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret (%vecstruct) %0, <4 x i32> %a, <4 x i32> %b)
+  ret void
+}
+; CHECK-SPIRV:         [[a_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV:         [[b_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]]
+; CHECK-SPIRV:     [[entry_3:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:      [[var_51:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_vecstruct]] Function
+; CHECK-SPIRV:      [[var_52:%[a-z0-9_]+]] = OpISubBorrow [[vecstruct]] [[a_3]] [[b_3]]
+; CHECK-SPIRV:                               OpStore [[var_51]] [[var_52]] 
+; CHECK-SPIRV:                               OpReturn
+; CHECK-SPIRV:                               OpFunctionEnd
+
+%struct.anon = type { i32, i32 }
+
+define spir_func void @test_builtin_isubborrow_anon(i32 %a, i32 %b) {
+  entry:
+  %0 = alloca %struct.anon
+  %1 = addrspacecast ptr %0 to ptr addrspace(4)
+  call spir_func void @_Z18__spirv_ISubBorrowIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4 %1, i32 %a, i32 %b)
+  ret void
+}
+; CHECK-SPIRV:        [[a_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV:        [[b_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]]
+; CHECK-SPIRV:    [[entry_4:%[a-z0-9_]+]] = OpLabel
+; CHECK-SPIRV:     [[var_59:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_struct_anon]] Function
+; CHECK-SPIRV:     [[var_61:%[a-z0-9_]+]] = OpPtrCastToGeneric [[_ptr_Generic_struct_anon]] [[var_59]]
+; CHECK-SPIRV:     [[var_62:%[a-z0-9_]+]] = OpISubBorrow [[struct_anon]] [[a_4]] [[b_4]]
+; CHECK-SPIRV:                              OpStore [[var_61]] [[var_62]]
+
+declare void @_Z18__spirv_ISubBorrowIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32)
+declare void @_Z18__spirv_ISubBorrowcc(ptr sret(%i8struct), i8, i8)
+declare void @_Z18__spirv_ISubBorrowss(ptr sret(%i16struct), i16, i16)
+declare void @_Z18__spirv_ISubBorrowii(ptr sret(%i32struct), i32, i32)
+declare void @_Z18__spirv_ISubBorrowll(ptr sret(%i64struct), i64, i64)
+declare void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret (%vecstruct), <4 x i32>, <4 x i32>)

>From 980f2b6ad0a92d98cc52325d7434da7a8ab58137 Mon Sep 17 00:00:00 2001
From: "Levytskyy, Vyacheslav" <vyacheslav.levytskyy at intel.com>
Date: Tue, 12 Nov 2024 03:19:14 -0800
Subject: [PATCH 5/5] LLVM IR add/sub instructions result in logical SPIR-V
 instructions when applied to bool type

---
 llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp   | 11 ++++++
 .../instructions/scalar-integer-arithmetic.ll | 34 +++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index d9a79d7e525b4b..59a1bf50b771b9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -414,6 +414,17 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
         validateForwardCalls(STI, MRI, GR, MI);
         break;
 
+      // ensure that LLVM IR add/sub instructions result in logical SPIR-V
+      // instructions when applied to bool type
+      case SPIRV::OpIAddS:
+      case SPIRV::OpIAddV:
+      case SPIRV::OpISubS:
+      case SPIRV::OpISubV:
+        if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
+                                      SPIRV::OpTypeBool))
+          MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));
+        break;
+
       // ensure that LLVM IR bitwise instructions result in logical SPIR-V
       // instructions when applied to bool type
       case SPIRV::OpBitwiseOrS:
diff --git a/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll b/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
index d222dfa570cf78..411866a67cc67b 100644
--- a/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
+++ b/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll
@@ -1,5 +1,11 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
+; CHECK-DAG: OpName [[BOOL_ADD:%.+]] "bool_add"
+; CHECK-DAG: OpName [[BOOL_SUB:%.+]] "bool_sub"
 ; CHECK-DAG: OpName [[SCALAR_ADD:%.+]] "scalar_add"
 ; CHECK-DAG: OpName [[SCALAR_SUB:%.+]] "scalar_sub"
 ; CHECK-DAG: OpName [[SCALAR_MUL:%.+]] "scalar_mul"
@@ -10,13 +16,28 @@
 
 ; CHECK-NOT: DAG-FENCE
 
+; CHECK-DAG: [[BOOL:%.+]] = OpTypeBool
 ; CHECK-DAG: [[SCALAR:%.+]] = OpTypeInt 32
 ; CHECK-DAG: [[SCALAR_FN:%.+]] = OpTypeFunction [[SCALAR]] [[SCALAR]] [[SCALAR]]
+; CHECK-DAG: [[BOOL_FN:%.+]] = OpTypeFunction [[BOOL]] [[BOOL]] [[BOOL]]
 
 ; CHECK-NOT: DAG-FENCE
 
 
 ;; Test add on scalar:
+define i1 @bool_add(i1 %a, i1 %b) {
+    %c = add i1 %a, %b
+    ret i1 %c
+}
+
+; CHECK:      [[BOOL_ADD]] = OpFunction [[BOOL]] None [[BOOL_FN]]
+; CHECK-NEXT: [[A:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK-NEXT: [[B:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK:      OpLabel
+; CHECK:      [[C:%.+]] = OpLogicalNotEqual [[BOOL]] [[A]] [[B]]
+; CHECK:      OpReturnValue [[C]]
+; CHECK-NEXT: OpFunctionEnd
+
 define i32 @scalar_add(i32 %a, i32 %b) {
     %c = add i32 %a, %b
     ret i32 %c
@@ -32,6 +53,19 @@ define i32 @scalar_add(i32 %a, i32 %b) {
 
 
 ;; Test sub on scalar:
+define i1 @bool_sub(i1 %a, i1 %b) {
+    %c = sub i1 %a, %b
+    ret i1 %c
+}
+
+; CHECK:      [[BOOL_SUB]] = OpFunction [[BOOL]] None [[BOOL_FN]]
+; CHECK-NEXT: [[A:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK-NEXT: [[B:%.+]] = OpFunctionParameter [[BOOL]]
+; CHECK:      OpLabel
+; CHECK:      [[C:%.+]] = OpLogicalNotEqual [[BOOL]] [[A]] [[B]]
+; CHECK:      OpReturnValue [[C]]
+; CHECK-NEXT: OpFunctionEnd
+
 define i32 @scalar_sub(i32 %a, i32 %b) {
     %c = sub i32 %a, %b
     ret i32 %c



More information about the llvm-commits mailing list