[llvm] [DirectX] Generalize DXIL integer legalization (PR #225233)

[Farzon Lotfi via llvm-commits]

================
@@ -25,6 +27,335 @@
 
 using namespace llvm;
 
+// Map an unsupported integer type to the smallest legal DXIL carrier type.
+static IntegerType *getLegalIntegerType(Type *Ty) {
+  auto *IntTy = dyn_cast<IntegerType>(Ty);
+  if (!IntTy)
+    return nullptr;
+
+  unsigned Width = IntTy->getBitWidth();
+  // DXIL uses i1 for SSA predicates even though booleans occupy i32 in memory.
+  if (Width == 1 || Width == 16 || Width == 32 || Width == 64)
+    return nullptr;
+  if (Width < 32)
+    return Type::getInt32Ty(Ty->getContext());
+  if (Width < 64)
+    return Type::getInt64Ty(Ty->getContext());
+  return nullptr;
+}
+
+enum class IntegerExtension { None, Zero, Sign };
+
+// Zero-extend the low Width bits of a legal-width carrier.
+static Value *maskToIntegerWidth(Value *V, unsigned Width,
+                                 IRBuilder<> &Builder) {
+  auto *LegalTy = cast<IntegerType>(V->getType());
+  APInt Mask = APInt::getLowBitsSet(LegalTy->getBitWidth(), Width);
+  return Builder.CreateAnd(V, ConstantInt::get(LegalTy, Mask));
+}
+
+// Return true when V is already a zero-extended Width-bit value.
+static bool isKnownZeroExtendedFromWidth(Value *V, unsigned Width,
+                                         const DataLayout &DL) {
+  unsigned LegalWidth = V->getType()->getIntegerBitWidth();
+  ConstantRange Range = computeConstantRangeIncludingKnownBits(
+      V, /*ForSigned=*/false, SimplifyQuery(DL, dyn_cast<Instruction>(V)));
+  return Range.getUnsignedMax().ult(APInt::getOneBitSet(LegalWidth, Width));
+}
+
+// Return true when V is already a sign-extended Width-bit value.
+static bool isKnownSignExtendedFromWidth(Value *V, unsigned Width,
+                                         const DataLayout &DL) {
+  unsigned LegalWidth = V->getType()->getIntegerBitWidth();
+  ConstantRange Range = computeConstantRangeIncludingKnownBits(
+      V, /*ForSigned=*/true, SimplifyQuery(DL, dyn_cast<Instruction>(V)));
+  APInt SignedMin = APInt::getSignedMinValue(Width).sext(LegalWidth);
+  APInt SignedMax = APInt::getSignedMaxValue(Width).sext(LegalWidth);
+  return Range.getSignedMin().sge(SignedMin) &&
+         Range.getSignedMax().sle(SignedMax);
+}
+
+// Get an operand's legal carrier, normalizing it only when a consumer requires
+// signed or unsigned narrow-integer semantics.
+static Value *
+getLegalizedIntegerOperand(Value *Operand, IntegerType *LegalTy,
+                           IntegerExtension Extension, IRBuilder<> &Builder,
+                           DenseMap<Value *, Value *> &ReplacedValues,
+                           const DataLayout &DL) {
+  if (Value *Replacement = ReplacedValues.lookup(Operand)) {
+    Replacement = Builder.CreateZExtOrTrunc(Replacement, LegalTy);
+    if (Extension == IntegerExtension::None)
+      return Replacement;
+    unsigned Width = cast<IntegerType>(Operand->getType())->getBitWidth();
+    if (Extension == IntegerExtension::Zero) {
+      if (auto *OverflowingOp = dyn_cast<OverflowingBinaryOperator>(Operand);
+          OverflowingOp && OverflowingOp->hasNoUnsignedWrap())
+        return Replacement;
+      return isKnownZeroExtendedFromWidth(Replacement, Width, DL)
+                 ? Replacement
+                 : maskToIntegerWidth(Replacement, Width, Builder);
+    }
+    if (isKnownSignExtendedFromWidth(Replacement, Width, DL))
+      return Replacement;
+    if (auto *OverflowingOp = dyn_cast<OverflowingBinaryOperator>(Operand);
+        OverflowingOp && OverflowingOp->hasNoSignedWrap())
+      return Replacement;
+    unsigned Shift = LegalTy->getBitWidth() - Width;
+    return Builder.CreateAShr(Builder.CreateShl(Replacement, Shift), Shift);
+  }
+  if (auto *C = dyn_cast<ConstantInt>(Operand))
+    return ConstantInt::get(
+        LegalTy, Extension == IntegerExtension::Sign
+                     ? C->getValue().sextOrTrunc(LegalTy->getBitWidth())
+                     : C->getValue().zextOrTrunc(LegalTy->getBitWidth()));
+  return Operand->getType() == LegalTy ? Operand : nullptr;
+}
+
+// bitcast <N x iM> to illegal iK -> extract and pack into an i32/i64 carrier.
+static bool
+legalizeNonStandardIntegerBitCast(BitCastInst &BitCast,
+                                  SmallVectorImpl<Instruction *> &ToRemove,
+                                  DenseMap<Value *, Value *> &ReplacedValues) {
+  auto *LegalTy = getLegalIntegerType(BitCast.getDestTy());
+  auto *SourceTy = dyn_cast<FixedVectorType>(BitCast.getSrcTy());
+  if (!LegalTy || !SourceTy || !SourceTy->getElementType()->isIntegerTy() ||
+      SourceTy->getPrimitiveSizeInBits() !=
----------------
farzonl wrote:

another case where  I will instead now do `report_fatal_error` instead of ignoring the issue.

https://github.com/llvm/llvm-project/pull/225233